ae-tracking 0.1.0

package/dist/src/wiki/block-to-md.d.ts

@@ -0,0 +1,74 @@
+type TextRun = {
+    content: string;
+    text_element_style?: {
+        bold?: boolean;
+        italic?: boolean;
+        inline_code?: boolean;
+    };
+};
+type ElemArr = {
+    text_run?: TextRun;
+}[];
+export interface Block {
+    block_id: string;
+    block_type: number;
+    children: string[];
+    page?: {
+        elements: ElemArr;
+    };
+    text?: {
+        elements: ElemArr;
+    };
+    heading1?: {
+        elements: ElemArr;
+    };
+    heading2?: {
+        elements: ElemArr;
+    };
+    heading3?: {
+        elements: ElemArr;
+    };
+    heading4?: {
+        elements: ElemArr;
+    };
+    heading5?: {
+        elements: ElemArr;
+    };
+    heading6?: {
+        elements: ElemArr;
+    };
+    heading7?: {
+        elements: ElemArr;
+    };
+    heading8?: {
+        elements: ElemArr;
+    };
+    heading9?: {
+        elements: ElemArr;
+    };
+    bullet?: {
+        elements: ElemArr;
+    };
+    ordered?: {
+        elements: ElemArr;
+    };
+    code?: {
+        language?: string;
+        elements: ElemArr;
+    };
+    quote?: {
+        elements: ElemArr;
+    };
+    callout?: {
+        elements: ElemArr;
+    };
+    table?: {
+        rows: string[][][];
+    };
+}
+export interface ConvertResult {
+    markdown: string;
+    unsupported: number[];
+}
+export declare function blocksToMarkdown(blocks: Block[]): ConvertResult;
+export {};

package/dist/src/wiki/block-to-md.js

@@ -0,0 +1,88 @@
+function renderText(elements) {
+    if (!elements)
+        return '';
+    return elements.map((e) => {
+        const tr = e.text_run;
+        if (!tr)
+            return '';
+        let t = tr.content ?? '';
+        const s = tr.text_element_style ?? {};
+        if (s.inline_code)
+            t = `\`${t}\``;
+        if (s.bold)
+            t = `**${t}**`;
+        if (s.italic)
+            t = `*${t}*`;
+        return t;
+    }).join('');
+}
+export function blocksToMarkdown(blocks) {
+    const byId = new Map();
+    for (const b of blocks)
+        byId.set(b.block_id, b);
+    const root = blocks.find((b) => b.block_type === 1);
+    if (!root)
+        return { markdown: '', unsupported: [] };
+    const unsupported = new Set();
+    const out = [];
+    const renderBlock = (b) => {
+        const t = b.block_type;
+        if (t === 1) {
+            out.push(`# ${renderText(b.page?.elements)}`, '');
+        }
+        else if (t === 2) {
+            out.push(renderText(b.text?.elements), '');
+        }
+        else if (t >= 3 && t <= 11) {
+            const level = Math.min(t - 1, 6);
+            const key = `heading${t - 2}`;
+            const el = b[key]?.elements;
+            out.push(`${'#'.repeat(level)} ${renderText(el)}`, '');
+        }
+        else if (t === 12) {
+            out.push(`- ${renderText(b.bullet?.elements)}`);
+        }
+        else if (t === 13) {
+            out.push(`1. ${renderText(b.ordered?.elements)}`);
+        }
+        else if (t === 14) {
+            const lang = b.code?.language ?? '';
+            out.push('```' + lang, renderText(b.code?.elements), '```', '');
+        }
+        else if (t === 15) {
+            out.push(`> ${renderText(b.quote?.elements)}`, '');
+        }
+        else if (t === 17) {
+            out.push(`> **Note:** ${renderText(b.callout?.elements)}`, '');
+        }
+        else if (t === 31 || t === 32) {
+            const rows = b.table?.rows ?? [];
+            if (rows.length) {
+                const header = rows[0].map((c) => c.join(' ').trim());
+                out.push(`| ${header.join(' | ')} |`);
+                out.push(`|${header.map(() => '---').join('|')}|`);
+                for (const r of rows.slice(1)) {
+                    out.push(`| ${r.map((c) => c.join(' ').trim()).join(' | ')} |`);
+                }
+                out.push('');
+            }
+        }
+        else if (t === 27 || t === 28) {
+            // image/file — skip
+        }
+        else {
+            unsupported.add(t);
+            out.push(`<!-- unsupported block: ${t} -->`, '');
+        }
+        for (const cid of b.children ?? []) {
+            const c = byId.get(cid);
+            if (c)
+                renderBlock(c);
+        }
+    };
+    renderBlock(root);
+    return {
+        markdown: out.join('\n').replace(/\n{3,}/g, '\n\n').trimEnd() + '\n',
+        unsupported: [...unsupported].sort((a, b) => a - b),
+    };
+}

package/dist/src/wiki/crawler.d.ts

@@ -0,0 +1,3 @@
+import type { SiteNode, PageMeta } from './types.js';
+export declare function fetchSitemap(): Promise<SiteNode[]>;
+export declare function filterInstallationSubtree(nodes: SiteNode[]): PageMeta[];

package/dist/src/wiki/crawler.js

@@ -0,0 +1,71 @@
+import { request } from 'undici';
+import { slugify } from './slugify.js';
+const SITEMAP_URL = 'https://docs-v2.thinkingdata.cn/te-doc/5.0/zh-CN.json';
+const DOCS_PAGE_BASE = 'https://docs-v2.thinkingdata.cn/?version=v5.0&lan=zh-CN&code=';
+export async function fetchSitemap() {
+    const { body, statusCode } = await request(SITEMAP_URL);
+    if (statusCode !== 200)
+        throw new Error(`sitemap fetch failed: ${statusCode}`);
+    return (await body.json());
+}
+export function filterInstallationSubtree(nodes) {
+    const root = nodes.find((n) => n.code === 'installation_menu');
+    if (!root)
+        throw new Error('installation_menu not found in sitemap');
+    const byToken = new Map();
+    for (const n of nodes)
+        byToken.set(n.wikiToken, n);
+    const inSubtree = new Set([root.wikiToken]);
+    let changed = true;
+    while (changed) {
+        changed = false;
+        for (const n of nodes) {
+            if (!inSubtree.has(n.wikiToken) && inSubtree.has(n.parentWikiToken)) {
+                inSubtree.add(n.wikiToken);
+                changed = true;
+            }
+        }
+    }
+    const pages = [];
+    for (const n of nodes) {
+        if (!inSubtree.has(n.wikiToken) || n.wikiToken === root.wikiToken)
+            continue;
+        if (n.type !== 'docx')
+            continue;
+        const segs = [];
+        let cur = n;
+        while (cur && cur.wikiToken !== root.wikiToken) {
+            segs.unshift(slugify(cur.name, cur.wikiToken));
+            cur = cur.parentWikiToken ? byToken.get(cur.parentWikiToken) : undefined;
+        }
+        pages.push({
+            wikiToken: n.wikiToken,
+            code: n.code,
+            name: n.name,
+            updateTime: n.updateTime,
+            parentWikiToken: n.parentWikiToken,
+            pathSegments: segs,
+            outPath: `wiki/te-docs/raw/${segs.join('/')}.md`,
+            sourceUrl: `${DOCS_PAGE_BASE}${encodeURIComponent(n.code)}`,
+        });
+    }
+    // Disambiguate duplicate outPaths: keep first occurrence unchanged,
+    // append -<wikiToken.slice(0,6)> before .md for subsequent duplicates.
+    const byPath = new Map();
+    for (const p of pages)
+        byPath.set(p.outPath, (byPath.get(p.outPath) ?? 0) + 1);
+    const seen = new Set();
+    for (const p of pages) {
+        if ((byPath.get(p.outPath) ?? 0) > 1 && seen.has(p.outPath)) {
+            const suffix = `-${p.wikiToken.slice(0, 6)}`;
+            const newOut = p.outPath.replace(/\.md$/, `${suffix}.md`);
+            const lastSeg = p.pathSegments[p.pathSegments.length - 1] + suffix;
+            p.pathSegments = [...p.pathSegments.slice(0, -1), lastSeg];
+            p.outPath = newOut;
+        }
+        else {
+            seen.add(p.outPath);
+        }
+    }
+    return pages;
+}

package/dist/src/wiki/index-writer.d.ts

@@ -0,0 +1,4 @@
+import type { PageMeta, FetchReport } from './types.js';
+export declare function renderIndex(pages: PageMeta[]): string;
+export declare function renderLog(reports: FetchReport[], pages: PageMeta[]): string;
+export declare function renderSchema(): string;

package/dist/src/wiki/index-writer.js

@@ -0,0 +1,70 @@
+export function renderIndex(pages) {
+    const lines = ['# AE Docs Wiki — Index', '', '| 页面 | 分类路径 | 原始链接 |', '|---|---|---|'];
+    for (const p of pages) {
+        const rel = p.outPath.replace(/^wiki\/te-docs\//, '');
+        const cat = p.pathSegments.slice(0, -1).join(' / ') || '—';
+        lines.push(`| [${p.name}](${rel}) | ${cat} | [docs](${p.sourceUrl}) |`);
+    }
+    lines.push('');
+    return lines.join('\n');
+}
+export function renderLog(reports, pages) {
+    const ok = reports.filter((r) => r.status === 'ok');
+    const fail = reports.filter((r) => r.status === 'fail');
+    const unsupported = new Map();
+    for (const r of reports) {
+        for (const t of r.unsupportedBlocks ?? []) {
+            unsupported.set(t, (unsupported.get(t) ?? 0) + 1);
+        }
+    }
+    const lines = [
+        '# AE Docs Wiki — Build Log',
+        '',
+        `Built: ${new Date().toISOString()}`,
+        `Pages planned: ${pages.length}`,
+        `ok: ${ok.length}`,
+        `fail: ${fail.length}`,
+        '',
+        '## Failures',
+    ];
+    if (fail.length === 0) {
+        lines.push('(none)');
+    }
+    else {
+        for (const r of fail)
+            lines.push(`- ${r.wikiToken} → ${r.outPath}: ${r.reason ?? 'unknown'}`);
+    }
+    lines.push('', '## Unsupported blocks encountered');
+    if (unsupported.size === 0) {
+        lines.push('(none)');
+    }
+    else {
+        for (const [t, n] of [...unsupported.entries()].sort((a, b) => a[0] - b[0])) {
+            lines.push(`- unsupported type ${t}: ${n} occurrences`);
+        }
+    }
+    lines.push('');
+    return lines.join('\n');
+}
+export function renderSchema() {
+    return [
+        '# AE Docs Wiki Schema',
+        '',
+        '## Layout',
+        '- `index.md` — catalog of pages with raw paths and source URLs',
+        '- `raw/` — Feishu docx → markdown mirror (read-only, crawler-owned)',
+        '- `synthesis/` — topic pages authored by synthesizer agents',
+        '- `log.md` — last build report',
+        '',
+        '## Query flow for consuming skills',
+        '1. Read `index.md` first.',
+        '2. For topic overviews, prefer `synthesis/`.',
+        '3. For exact API / config lookup, read `raw/`.',
+        '4. Cite `sourceUrl` from page frontmatter when quoting.',
+        '',
+        '## Rules',
+        '- Do not edit `raw/` — it is regenerated by the crawler.',
+        '- Hand-authored or LLM-synthesized notes go in `synthesis/`.',
+        '',
+    ].join('\n');
+}

package/dist/src/wiki/slugify.d.ts

@@ -0,0 +1 @@
+export declare function slugify(name: string, fallbackToken?: string): string;

package/dist/src/wiki/slugify.js

@@ -0,0 +1,11 @@
+export function slugify(name, fallbackToken = '') {
+    const s = name
+        .toLowerCase()
+        .replace(/[\s/]+/g, '-')
+        .replace(/[^\p{L}\p{N}\-]+/gu, '')
+        .replace(/-+/g, '-')
+        .replace(/^-|-$/g, '');
+    if (s)
+        return s;
+    return fallbackToken.slice(0, 8);
+}

package/dist/src/wiki/types.d.ts

@@ -0,0 +1,29 @@
+export interface SiteNode {
+    code: string;
+    id: string;
+    name: string;
+    type: string;
+    updateTime: number;
+    wikiToken: string;
+    parentWikiToken: string;
+}
+export interface PageMeta {
+    wikiToken: string;
+    code: string;
+    name: string;
+    updateTime: number;
+    parentWikiToken: string;
+    /** slug-joined path segments derived from ancestor names */
+    pathSegments: string[];
+    /** canonical filesystem path: `wiki/te-docs/raw/<slug-path>.md` */
+    outPath: string;
+    /** original docs URL */
+    sourceUrl: string;
+}
+export interface FetchReport {
+    wikiToken: string;
+    outPath: string;
+    status: 'ok' | 'fail';
+    reason?: string;
+    unsupportedBlocks?: number[];
+}

package/dist/src/wiki/types.js

@@ -0,0 +1 @@
+export {};

package/dist/src/wiki-query/query.d.ts

@@ -0,0 +1,8 @@
+export declare function readSynthesis(slug: string): string | null;
+export interface IndexHit {
+    name: string;
+    rawPath: string;
+    sourceUrl: string;
+}
+export declare function searchIndex(keyword: string): IndexHit[];
+export declare function readRaw(relPath: string): string;

package/dist/src/wiki-query/query.js

@@ -0,0 +1,32 @@
+import { readFileSync, existsSync } from 'node:fs';
+import { join } from 'node:path';
+import { getPackageRoot } from '../cli/package-root.js';
+const WIKI_ROOT = join(getPackageRoot(), 'wiki/te-docs');
+export function readSynthesis(slug) {
+    const p = join(WIKI_ROOT, 'synthesis', `${slug}.md`);
+    if (!existsSync(p))
+        return null;
+    return readFileSync(p, 'utf8');
+}
+export function searchIndex(keyword) {
+    const indexPath = join(WIKI_ROOT, 'index.md');
+    if (!existsSync(indexPath))
+        return [];
+    const lines = readFileSync(indexPath, 'utf8').split('\n');
+    const kw = keyword.toLowerCase();
+    const hits = [];
+    for (const line of lines) {
+        const m = /^\|\s*\[([^\]]+)\]\(([^)]+)\)\s*\|.*\|\s*\[docs\]\(([^)]+)\)\s*\|$/.exec(line);
+        if (!m)
+            continue;
+        const [, name, rawPath, sourceUrl] = m;
+        if (name.toLowerCase().includes(kw) || rawPath.toLowerCase().includes(kw)) {
+            hits.push({ name, rawPath, sourceUrl });
+        }
+    }
+    return hits;
+}
+export function readRaw(relPath) {
+    const p = join(WIKI_ROOT, relPath);
+    return readFileSync(p, 'utf8');
+}

package/dist/src/xlsx/read.d.ts

@@ -0,0 +1,2 @@
+import type { Draft } from '../plan/types.js';
+export declare function readTemplateXlsx(filePath: string): Promise<Draft>;