inkdrift 0.1.0

package/dist/modules/distiller/test.js

@@ -0,0 +1,92 @@
+/**
+ * distiller module standalone test
+ *
+ * Usage: npx tsx src/modules/distiller/test.ts
+ *
+ * Reads a real article from FlowUs, extracts content, sends to LLM,
+ * prints distilled knowledge points.
+ */
+import '../../utils/config.js';
+import { config } from '../../utils/config.js';
+import { FlowUsClient } from '../flowus-source/index.js';
+import { extract } from '../extractor/index.js';
+import { distill } from './index.js';
+const { flowus, glm } = config;
+if (!flowus.token || !flowus.databaseId) {
+    console.error('Missing FLOWUS_TOKEN or FLOWUS_DATABASE_ID in .env');
+    process.exit(1);
+}
+if (!glm.apiKey) {
+    console.error('Missing GLM_API_KEY in .env');
+    process.exit(1);
+}
+const client = new FlowUsClient({
+    baseUrl: flowus.baseUrl,
+    token: flowus.token,
+    databaseId: flowus.databaseId,
+});
+console.log('=== distiller module test ===\n');
+// Step 1: Get one article from FlowUs
+console.log('[1] Fetching article from FlowUs...');
+const pages = await client.queryNewArticles();
+if (pages.length === 0) {
+    console.log('    No articles found. Nothing to distill.');
+    process.exit(0);
+}
+console.log(`    Found ${pages.length} article(s), using first one.\n`);
+const page = pages[0];
+const article = await client.getFullArticle(page);
+console.log(`    Title: ${article.title}`);
+console.log(`    URL: ${article.url}`);
+console.log(`    Raw content length: ${article.content.length} chars\n`);
+// Step 2: Extract (clean)
+console.log('[2] Running extractor...');
+const extracted = extract(article.content);
+console.log(`    Cleaned length: ${extracted.markdown.length} chars`);
+console.log(`    Word count: ${extracted.wordCount}\n`);
+// Step 3: Distill with LLM
+console.log('[3] Calling LLM for knowledge distillation...');
+console.log(`    Model: ${glm.model}\n`);
+const knowledgePoints = await distill(extracted.markdown, article.url, {
+    baseUrl: glm.baseUrl,
+    apiKey: glm.apiKey,
+    model: glm.model,
+});
+console.log(`    Extracted ${knowledgePoints.length} knowledge point(s)\n`);
+if (knowledgePoints.length === 0) {
+    console.log('    No knowledge points (article may have no substance).');
+    process.exit(0);
+}
+// Step 4: Print knowledge points
+console.log('[4] Knowledge points:');
+for (let i = 0; i < knowledgePoints.length; i++) {
+    const kp = knowledgePoints[i];
+    console.log(`\n    --- KP ${i + 1} ---`);
+    console.log(`    Title: ${kp.title}`);
+    console.log(`    Content: ${kp.content.substring(0, 150)}...`);
+    console.log(`    Tags: ${kp.tags.join(', ')}`);
+    console.log(`    Importance: ${kp.importance}`);
+}
+// Step 5: Validate
+console.log('\n[5] Validating...');
+const checks = [];
+for (const kp of knowledgePoints) {
+    if (!kp.content.includes(article.url)) {
+        checks.push(`FAIL: "${kp.title}" missing source URL`);
+    }
+    if (kp.tags.length < 2 || kp.tags.length > 5) {
+        checks.push(`FAIL: "${kp.title}" tags count ${kp.tags.length}`);
+    }
+    if (kp.importance < 0 || kp.importance > 1) {
+        checks.push(`FAIL: "${kp.title}" importance out of range`);
+    }
+}
+if (checks.length === 0) {
+    console.log('    PASS: All knowledge points valid');
+}
+else {
+    for (const c of checks)
+        console.log(`    ${c}`);
+}
+console.log('\n=== Test complete ===');
+//# sourceMappingURL=test.js.map

package/dist/modules/distiller/test.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"test.js","sourceRoot":"","sources":["../../../src/modules/distiller/test.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,uBAAuB,CAAC;AAC/B,OAAO,EAAE,MAAM,EAAE,MAAM,uBAAuB,CAAC;AAC/C,OAAO,EAAE,YAAY,EAAE,MAAM,2BAA2B,CAAC;AACzD,OAAO,EAAE,OAAO,EAAE,MAAM,uBAAuB,CAAC;AAChD,OAAO,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAErC,MAAM,EAAE,MAAM,EAAE,GAAG,EAAE,GAAG,MAAM,CAAC;AAE/B,IAAI,CAAC,MAAM,CAAC,KAAK,IAAI,CAAC,MAAM,CAAC,UAAU,EAAE,CAAC;IACxC,OAAO,CAAC,KAAK,CAAC,oDAAoD,CAAC,CAAC;IACpE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC;AAED,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE,CAAC;IAChB,OAAO,CAAC,KAAK,CAAC,6BAA6B,CAAC,CAAC;IAC7C,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC;AAED,MAAM,MAAM,GAAG,IAAI,YAAY,CAAC;IAC9B,OAAO,EAAE,MAAM,CAAC,OAAO;IACvB,KAAK,EAAE,MAAM,CAAC,KAAK;IACnB,UAAU,EAAE,MAAM,CAAC,UAAU;CAC9B,CAAC,CAAC;AAEH,OAAO,CAAC,GAAG,CAAC,iCAAiC,CAAC,CAAC;AAE/C,sCAAsC;AACtC,OAAO,CAAC,GAAG,CAAC,qCAAqC,CAAC,CAAC;AACnD,MAAM,KAAK,GAAG,MAAM,MAAM,CAAC,gBAAgB,EAAE,CAAC;AAC9C,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;IACvB,OAAO,CAAC,GAAG,CAAC,4CAA4C,CAAC,CAAC;IAC1D,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC;AACD,OAAO,CAAC,GAAG,CAAC,aAAa,KAAK,CAAC,MAAM,iCAAiC,CAAC,CAAC;AAExE,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;AACtB,MAAM,OAAO,GAAG,MAAM,MAAM,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC;AAElD,OAAO,CAAC,GAAG,CAAC,cAAc,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC;AAC3C,OAAO,CAAC,GAAG,CAAC,YAAY,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC;AACvC,OAAO,CAAC,GAAG,CAAC,2BAA2B,OAAO,CAAC,OAAO,CAAC,MAAM,UAAU,CAAC,CAAC;AAEzE,0BAA0B;AAC1B,OAAO,CAAC,GAAG,CAAC,0BAA0B,CAAC,CAAC;AACxC,MAAM,SAAS,GAAG,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;AAC3C,OAAO,CAAC,GAAG,CAAC,uBAAuB,SAAS,CAAC,QAAQ,CAAC,MAAM,QAAQ,CAAC,CAAC;AACtE,OAAO,CAAC,GAAG,CAAC,mBAAmB,SAAS,CAAC,SAAS,IAAI,CAAC,CAAC;AAExD,2BAA2B;AAC3B,OAAO,CAAC,GAAG,CAAC,+CAA+C,CAAC,CAAC;AAC7D,OAAO,CAAC,GAAG,CAAC,cAAc,GAAG,CAAC,KAAK,IAAI,CAAC,CAAC;AAEzC,MAAM,eAAe,GAAG,MAAM,OAAO,CAAC,SAAS,CAAC,QAAQ,EAAE,OAAO,CAAC,GAAG,EAAE;IACrE,OAAO,EAAE,GAAG,CAAC,OAAO;IACpB,MAAM,EAAE,GAAG,CAAC,MAAM;IAClB,KAAK,EAAE,GAAG,CAAC,KAAK;CACjB,CAAC,CAAC;AAEH,OAAO,CAAC,GAAG,CAAC,iBAAiB,eAAe,CAAC,MAAM,uBAAuB,CAAC,CAAC;AAE5E,IAAI,eAAe,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;IACjC,OAAO,CAAC,GAAG,CAAC,0DAA0D,CAAC,CAAC;IACxE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC;AAED,iCAAiC;AACjC,OAAO,CAAC,GAAG,CAAC,uBAAuB,CAAC,CAAC;AACrC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,eAAe,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;IAChD,MAAM,EAAE,GAAG,eAAe,CAAC,CAAC,CAAC,CAAC;IAC9B,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;IACzC,OAAO,CAAC,GAAG,CAAC,cAAc,EAAE,CAAC,KAAK,EAAE,CAAC,CAAC;IACtC,OAAO,CAAC,GAAG,CAAC,gBAAgB,EAAE,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,EAAE,GAAG,CAAC,KAAK,CAAC,CAAC;IAC/D,OAAO,CAAC,GAAG,CAAC,aAAa,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAC/C,OAAO,CAAC,GAAG,CAAC,mBAAmB,EAAE,CAAC,UAAU,EAAE,CAAC,CAAC;AAClD,CAAC;AAED,mBAAmB;AACnB,OAAO,CAAC,GAAG,CAAC,qBAAqB,CAAC,CAAC;AACnC,MAAM,MAAM,GAAa,EAAE,CAAC;AAE5B,KAAK,MAAM,EAAE,IAAI,eAAe,EAAE,CAAC;IACjC,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC;QACtC,MAAM,CAAC,IAAI,CAAC,UAAU,EAAE,CAAC,KAAK,sBAAsB,CAAC,CAAC;IACxD,CAAC;IACD,IAAI,EAAE,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC7C,MAAM,CAAC,IAAI,CAAC,UAAU,EAAE,CAAC,KAAK,gBAAgB,EAAE,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;IAClE,CAAC;IACD,IAAI,EAAE,CAAC,UAAU,GAAG,CAAC,IAAI,EAAE,CAAC,UAAU,GAAG,CAAC,EAAE,CAAC;QAC3C,MAAM,CAAC,IAAI,CAAC,UAAU,EAAE,CAAC,KAAK,2BAA2B,CAAC,CAAC;IAC7D,CAAC;AACH,CAAC;AAED,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;IACxB,OAAO,CAAC,GAAG,CAAC,sCAAsC,CAAC,CAAC;AACtD,CAAC;KAAM,CAAC;IACN,KAAK,MAAM,CAAC,IAAI,MAAM;QAAE,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;AAClD,CAAC;AAED,OAAO,CAAC,GAAG,CAAC,yBAAyB,CAAC,CAAC"}

package/dist/modules/extractor/index.d.ts

@@ -0,0 +1,52 @@
+/**
+ * extractor module
+ *
+ * Receives Markdown from flowus-source, cleans it up,
+ * extracts metadata (summary, word count), and splits into sections.
+ *
+ * Pure text processing, no external dependencies.
+ */
+export interface Section {
+    heading: string;
+    content: string;
+}
+export interface ExtractedContent {
+    markdown: string;
+    summary: string;
+    wordCount: number;
+    sections: Section[];
+}
+/**
+ * Detect if content is a FlowUs capture failure placeholder.
+ * Returns true if the content contains known placeholder patterns
+ * AND is short (under 500 chars), indicating FlowUs couldn't fetch the real content.
+ */
+export declare function isPlaceholderContent(content: string): boolean;
+/**
+ * Extract the first URL from content (for placeholder articles,
+ * this is typically the original WeChat article link).
+ */
+export declare function extractUrl(content: string): string | null;
+/**
+ * Clean Markdown: merge blank lines, trim whitespace, remove blank-only lines.
+ * Preserves code blocks, lists, and other formatting.
+ */
+export declare function cleanMarkdown(md: string): string;
+/**
+ * Extract metadata: summary (first 200 chars of body text) and word count.
+ * Word count excludes headings and blank lines.
+ */
+export declare function extractMetadata(md: string): {
+    summary: string;
+    wordCount: number;
+};
+/**
+ * Split Markdown into sections by headings (# ~ ###).
+ * The first section has an empty heading if content precedes the first heading.
+ */
+export declare function splitSections(md: string): Section[];
+/**
+ * One-shot: clean + extract metadata + split sections.
+ */
+export declare function extract(md: string): ExtractedContent;
+//# sourceMappingURL=index.d.ts.map

package/dist/modules/extractor/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/modules/extractor/index.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAIH,MAAM,WAAW,OAAO;IACtB,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,EAAE,OAAO,EAAE,CAAC;CACrB;AAWD;;;;GAIG;AACH,wBAAgB,oBAAoB,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAI7D;AAED;;;GAGG;AACH,wBAAgB,UAAU,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAGzD;AAID;;;GAGG;AACH,wBAAgB,aAAa,CAAC,EAAE,EAAE,MAAM,GAAG,MAAM,CAsBhD;AAED;;;GAGG;AACH,wBAAgB,eAAe,CAAC,EAAE,EAAE,MAAM,GAAG;IAAE,OAAO,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,MAAM,CAAA;CAAE,CAUlF;AAED;;;GAGG;AACH,wBAAgB,aAAa,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,EAAE,CA+BnD;AAED;;GAEG;AACH,wBAAgB,OAAO,CAAC,EAAE,EAAE,MAAM,GAAG,gBAAgB,CAMpD"}

package/dist/modules/extractor/index.js

@@ -0,0 +1,114 @@
+/**
+ * extractor module
+ *
+ * Receives Markdown from flowus-source, cleans it up,
+ * extracts metadata (summary, word count), and splits into sections.
+ *
+ * Pure text processing, no external dependencies.
+ */
+// ---- Placeholder detection ----
+const PLACEHOLDER_PATTERNS = [
+    '微信安全验证',
+    '无法自动获取',
+    '请在微信客户端打开',
+    '内容无法访问',
+];
+/**
+ * Detect if content is a FlowUs capture failure placeholder.
+ * Returns true if the content contains known placeholder patterns
+ * AND is short (under 500 chars), indicating FlowUs couldn't fetch the real content.
+ */
+export function isPlaceholderContent(content) {
+    const trimmed = content.trim();
+    if (trimmed.length > 500)
+        return false;
+    return PLACEHOLDER_PATTERNS.some((p) => trimmed.includes(p));
+}
+/**
+ * Extract the first URL from content (for placeholder articles,
+ * this is typically the original WeChat article link).
+ */
+export function extractUrl(content) {
+    const match = content.match(/https?:\/\/[^\s)\]}>]+/);
+    return match ? match[0] : null;
+}
+// ---- Public API ----
+/**
+ * Clean Markdown: merge blank lines, trim whitespace, remove blank-only lines.
+ * Preserves code blocks, lists, and other formatting.
+ */
+export function cleanMarkdown(md) {
+    const lines = md.split('\n');
+    const result = [];
+    for (let i = 0; i < lines.length; i++) {
+        const trimmed = lines[i].trimEnd();
+        // Keep the line as-is (with leading whitespace for indented content)
+        // but skip lines that are purely whitespace
+        if (trimmed === '' && result.length > 0 && result[result.length - 1] === '') {
+            continue; // skip consecutive blank lines
+        }
+        result.push(trimmed);
+    }
+    // Trim trailing blank lines
+    while (result.length > 0 && result[result.length - 1] === '') {
+        result.pop();
+    }
+    return result.join('\n');
+}
+/**
+ * Extract metadata: summary (first 200 chars of body text) and word count.
+ * Word count excludes headings and blank lines.
+ */
+export function extractMetadata(md) {
+    const bodyLines = md
+        .split('\n')
+        .filter((line) => line.trim() !== '' && !line.startsWith('#'));
+    const bodyText = bodyLines.join(' ').replace(/\s+/g, ' ').trim();
+    const wordCount = bodyText.length;
+    const summary = bodyText.substring(0, 200);
+    return { summary, wordCount };
+}
+/**
+ * Split Markdown into sections by headings (# ~ ###).
+ * The first section has an empty heading if content precedes the first heading.
+ */
+export function splitSections(md) {
+    const lines = md.split('\n');
+    const sections = [];
+    let currentHeading = '';
+    let currentLines = [];
+    for (const line of lines) {
+        if (/^#{1,3}\s/.test(line)) {
+            // Flush current section
+            if (currentLines.length > 0 || currentHeading !== '') {
+                sections.push({
+                    heading: currentHeading,
+                    content: currentLines.join('\n').trim(),
+                });
+            }
+            currentHeading = line.replace(/^#+\s*/, '').trim();
+            currentLines = [];
+        }
+        else {
+            currentLines.push(line);
+        }
+    }
+    // Flush last section
+    if (currentLines.length > 0 || currentHeading !== '') {
+        sections.push({
+            heading: currentHeading,
+            content: currentLines.join('\n').trim(),
+        });
+    }
+    return sections;
+}
+/**
+ * One-shot: clean + extract metadata + split sections.
+ */
+export function extract(md) {
+    const markdown = cleanMarkdown(md);
+    const { summary, wordCount } = extractMetadata(markdown);
+    const sections = splitSections(markdown);
+    return { markdown, summary, wordCount, sections };
+}
+//# sourceMappingURL=index.js.map

package/dist/modules/extractor/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/modules/extractor/index.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAgBH,kCAAkC;AAElC,MAAM,oBAAoB,GAAG;IAC3B,QAAQ;IACR,QAAQ;IACR,WAAW;IACX,QAAQ;CACT,CAAC;AAEF;;;;GAIG;AACH,MAAM,UAAU,oBAAoB,CAAC,OAAe;IAClD,MAAM,OAAO,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC;IAC/B,IAAI,OAAO,CAAC,MAAM,GAAG,GAAG;QAAE,OAAO,KAAK,CAAC;IACvC,OAAO,oBAAoB,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC;AAC/D,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,UAAU,CAAC,OAAe;IACxC,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,wBAAwB,CAAC,CAAC;IACtD,OAAO,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;AACjC,CAAC;AAED,uBAAuB;AAEvB;;;GAGG;AACH,MAAM,UAAU,aAAa,CAAC,EAAU;IACtC,MAAM,KAAK,GAAG,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAC7B,MAAM,MAAM,GAAa,EAAE,CAAC;IAE5B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACtC,MAAM,OAAO,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC;QAEnC,qEAAqE;QACrE,4CAA4C;QAC5C,IAAI,OAAO,KAAK,EAAE,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,IAAI,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC;YAC5E,SAAS,CAAC,+BAA+B;QAC3C,CAAC;QAED,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IACvB,CAAC;IAED,4BAA4B;IAC5B,OAAO,MAAM,CAAC,MAAM,GAAG,CAAC,IAAI,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC;QAC7D,MAAM,CAAC,GAAG,EAAE,CAAC;IACf,CAAC;IAED,OAAO,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC3B,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,eAAe,CAAC,EAAU;IACxC,MAAM,SAAS,GAAG,EAAE;SACjB,KAAK,CAAC,IAAI,CAAC;SACX,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,KAAK,EAAE,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC;IAEjE,MAAM,QAAQ,GAAG,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;IACjE,MAAM,SAAS,GAAG,QAAQ,CAAC,MAAM,CAAC;IAClC,MAAM,OAAO,GAAG,QAAQ,CAAC,SAAS,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;IAE3C,OAAO,EAAE,OAAO,EAAE,SAAS,EAAE,CAAC;AAChC,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,aAAa,CAAC,EAAU;IACtC,MAAM,KAAK,GAAG,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAC7B,MAAM,QAAQ,GAAc,EAAE,CAAC;IAC/B,IAAI,cAAc,GAAG,EAAE,CAAC;IACxB,IAAI,YAAY,GAAa,EAAE,CAAC;IAEhC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,IAAI,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;YAC3B,wBAAwB;YACxB,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,IAAI,cAAc,KAAK,EAAE,EAAE,CAAC;gBACrD,QAAQ,CAAC,IAAI,CAAC;oBACZ,OAAO,EAAE,cAAc;oBACvB,OAAO,EAAE,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE;iBACxC,CAAC,CAAC;YACL,CAAC;YACD,cAAc,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;YACnD,YAAY,GAAG,EAAE,CAAC;QACpB,CAAC;aAAM,CAAC;YACN,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC1B,CAAC;IACH,CAAC;IAED,qBAAqB;IACrB,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,IAAI,cAAc,KAAK,EAAE,EAAE,CAAC;QACrD,QAAQ,CAAC,IAAI,CAAC;YACZ,OAAO,EAAE,cAAc;YACvB,OAAO,EAAE,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE;SACxC,CAAC,CAAC;IACL,CAAC;IAED,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,OAAO,CAAC,EAAU;IAChC,MAAM,QAAQ,GAAG,aAAa,CAAC,EAAE,CAAC,CAAC;IACnC,MAAM,EAAE,OAAO,EAAE,SAAS,EAAE,GAAG,eAAe,CAAC,QAAQ,CAAC,CAAC;IACzD,MAAM,QAAQ,GAAG,aAAa,CAAC,QAAQ,CAAC,CAAC;IAEzC,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,SAAS,EAAE,QAAQ,EAAE,CAAC;AACpD,CAAC"}

package/dist/modules/extractor/test.d.ts

@@ -0,0 +1,10 @@
+/**
+ * extractor module standalone test
+ *
+ * Usage: npx tsx src/modules/extractor/test.ts
+ *
+ * Reads a real article from FlowUs, passes through extractor,
+ * prints cleaned result, summary, word count, sections.
+ */
+import '../../utils/config.js';
+//# sourceMappingURL=test.d.ts.map

package/dist/modules/extractor/test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"test.d.ts","sourceRoot":"","sources":["../../../src/modules/extractor/test.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,uBAAuB,CAAC"}

package/dist/modules/extractor/test.js

@@ -0,0 +1,80 @@
+/**
+ * extractor module standalone test
+ *
+ * Usage: npx tsx src/modules/extractor/test.ts
+ *
+ * Reads a real article from FlowUs, passes through extractor,
+ * prints cleaned result, summary, word count, sections.
+ */
+import '../../utils/config.js';
+import { config } from '../../utils/config.js';
+import { FlowUsClient } from '../flowus-source/index.js';
+import { extract } from './index.js';
+const { token, databaseId, baseUrl } = config.flowus;
+if (!token || !databaseId) {
+    console.error('Missing FLOWUS_TOKEN or FLOWUS_DATABASE_ID in .env');
+    process.exit(1);
+}
+const client = new FlowUsClient({ baseUrl, token, databaseId });
+console.log('=== extractor module test ===\n');
+// Step 1: Get one article from FlowUs
+console.log('[1] Fetching articles from FlowUs...');
+const pages = await client.queryNewArticles();
+if (pages.length === 0) {
+    console.log('    No articles found. Test passed (nothing to extract).');
+    process.exit(0);
+}
+console.log(`    Found ${pages.length} article(s), using first one.\n`);
+const page = pages[0];
+const article = await client.getFullArticle(page);
+console.log(`    Title: ${article.title}`);
+console.log(`    Raw content length: ${article.content.length} chars\n`);
+// Step 2: Extract
+console.log('[2] Running extract()...');
+const result = extract(article.content);
+console.log(`    Cleaned length: ${result.markdown.length} chars`);
+console.log(`    Word count: ${result.wordCount}`);
+console.log(`    Summary (${result.summary.length} chars):`);
+console.log(`      ${result.summary.substring(0, 120)}...`);
+console.log(`    Sections: ${result.sections.length}`);
+for (const s of result.sections.slice(0, 5)) {
+    const label = s.heading || '(preamble)';
+    console.log(`      - "${label}" (${s.content.length} chars)`);
+}
+if (result.sections.length > 5) {
+    console.log(`      ... and ${result.sections.length - 5} more`);
+}
+// Step 3: Show cleaned content preview
+console.log('\n[3] Cleaned content preview (first 500 chars):');
+console.log('---');
+console.log(result.markdown.substring(0, 500));
+console.log('---');
+// Step 4: Verify basic clean rules
+console.log('\n[4] Verifying clean rules...');
+const checks = [];
+// No consecutive blank lines
+if (!/\n\n\n/.test(result.markdown)) {
+    checks.push('PASS: No consecutive blank lines');
+}
+else {
+    checks.push('FAIL: Found consecutive blank lines');
+}
+// No trailing whitespace per line
+const trailingWs = result.markdown.split('\n').filter((l) => l !== l.trimEnd());
+if (trailingWs.length === 0) {
+    checks.push('PASS: No trailing whitespace');
+}
+else {
+    checks.push(`FAIL: ${trailingWs.length} lines with trailing whitespace`);
+}
+// No leading/trailing blank lines
+if (result.markdown === result.markdown.trim()) {
+    checks.push('PASS: No leading/trailing blank lines');
+}
+else {
+    checks.push('FAIL: Has leading or trailing blank lines');
+}
+for (const c of checks)
+    console.log(`    ${c}`);
+console.log('\n=== Test complete ===');
+//# sourceMappingURL=test.js.map

package/dist/modules/extractor/test.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"test.js","sourceRoot":"","sources":["../../../src/modules/extractor/test.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,uBAAuB,CAAC;AAC/B,OAAO,EAAE,MAAM,EAAE,MAAM,uBAAuB,CAAC;AAC/C,OAAO,EAAE,YAAY,EAAE,MAAM,2BAA2B,CAAC;AACzD,OAAO,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAErC,MAAM,EAAE,KAAK,EAAE,UAAU,EAAE,OAAO,EAAE,GAAG,MAAM,CAAC,MAAM,CAAC;AAErD,IAAI,CAAC,KAAK,IAAI,CAAC,UAAU,EAAE,CAAC;IAC1B,OAAO,CAAC,KAAK,CAAC,oDAAoD,CAAC,CAAC;IACpE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC;AAED,MAAM,MAAM,GAAG,IAAI,YAAY,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE,UAAU,EAAE,CAAC,CAAC;AAEhE,OAAO,CAAC,GAAG,CAAC,iCAAiC,CAAC,CAAC;AAE/C,sCAAsC;AACtC,OAAO,CAAC,GAAG,CAAC,sCAAsC,CAAC,CAAC;AACpD,MAAM,KAAK,GAAG,MAAM,MAAM,CAAC,gBAAgB,EAAE,CAAC;AAC9C,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;IACvB,OAAO,CAAC,GAAG,CAAC,0DAA0D,CAAC,CAAC;IACxE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC;AACD,OAAO,CAAC,GAAG,CAAC,aAAa,KAAK,CAAC,MAAM,iCAAiC,CAAC,CAAC;AAExE,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;AACtB,MAAM,OAAO,GAAG,MAAM,MAAM,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC;AAElD,OAAO,CAAC,GAAG,CAAC,cAAc,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC;AAC3C,OAAO,CAAC,GAAG,CAAC,2BAA2B,OAAO,CAAC,OAAO,CAAC,MAAM,UAAU,CAAC,CAAC;AAEzE,kBAAkB;AAClB,OAAO,CAAC,GAAG,CAAC,0BAA0B,CAAC,CAAC;AACxC,MAAM,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;AAExC,OAAO,CAAC,GAAG,CAAC,uBAAuB,MAAM,CAAC,QAAQ,CAAC,MAAM,QAAQ,CAAC,CAAC;AACnE,OAAO,CAAC,GAAG,CAAC,mBAAmB,MAAM,CAAC,SAAS,EAAE,CAAC,CAAC;AACnD,OAAO,CAAC,GAAG,CAAC,gBAAgB,MAAM,CAAC,OAAO,CAAC,MAAM,UAAU,CAAC,CAAC;AAC7D,OAAO,CAAC,GAAG,CAAC,SAAS,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,EAAE,GAAG,CAAC,KAAK,CAAC,CAAC;AAC5D,OAAO,CAAC,GAAG,CAAC,iBAAiB,MAAM,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC;AAEvD,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC;IAC5C,MAAM,KAAK,GAAG,CAAC,CAAC,OAAO,IAAI,YAAY,CAAC;IACxC,OAAO,CAAC,GAAG,CAAC,YAAY,KAAK,MAAM,CAAC,CAAC,OAAO,CAAC,MAAM,SAAS,CAAC,CAAC;AAChE,CAAC;AACD,IAAI,MAAM,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;IAC/B,OAAO,CAAC,GAAG,CAAC,iBAAiB,MAAM,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,OAAO,CAAC,CAAC;AAClE,CAAC;AAED,uCAAuC;AACvC,OAAO,CAAC,GAAG,CAAC,kDAAkD,CAAC,CAAC;AAChE,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;AACnB,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC;AAC/C,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;AAEnB,mCAAmC;AACnC,OAAO,CAAC,GAAG,CAAC,gCAAgC,CAAC,CAAC;AAC9C,MAAM,MAAM,GAAa,EAAE,CAAC;AAE5B,6BAA6B;AAC7B,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC;IACpC,MAAM,CAAC,IAAI,CAAC,kCAAkC,CAAC,CAAC;AAClD,CAAC;KAAM,CAAC;IACN,MAAM,CAAC,IAAI,CAAC,qCAAqC,CAAC,CAAC;AACrD,CAAC;AAED,kCAAkC;AAClC,MAAM,UAAU,GAAG,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC;AAChF,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;IAC5B,MAAM,CAAC,IAAI,CAAC,8BAA8B,CAAC,CAAC;AAC9C,CAAC;KAAM,CAAC;IACN,MAAM,CAAC,IAAI,CAAC,SAAS,UAAU,CAAC,MAAM,iCAAiC,CAAC,CAAC;AAC3E,CAAC;AAED,kCAAkC;AAClC,IAAI,MAAM,CAAC,QAAQ,KAAK,MAAM,CAAC,QAAQ,CAAC,IAAI,EAAE,EAAE,CAAC;IAC/C,MAAM,CAAC,IAAI,CAAC,uCAAuC,CAAC,CAAC;AACvD,CAAC;KAAM,CAAC;IACN,MAAM,CAAC,IAAI,CAAC,2CAA2C,CAAC,CAAC;AAC3D,CAAC;AAED,KAAK,MAAM,CAAC,IAAI,MAAM;IAAE,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;AAEhD,OAAO,CAAC,GAAG,CAAC,yBAAyB,CAAC,CAAC"}

package/dist/modules/fetcher/index.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=index.d.ts.map

package/dist/modules/fetcher/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/modules/fetcher/index.ts"],"names":[],"mappings":""}

package/dist/modules/fetcher/index.js

@@ -0,0 +1,3 @@
+export {};
+// fetcher module - article fetching
+//# sourceMappingURL=index.js.map

package/dist/modules/fetcher/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/modules/fetcher/index.ts"],"names":[],"mappings":";AAAA,oCAAoC"}

package/dist/modules/fetcher/test.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=test.d.ts.map

package/dist/modules/fetcher/test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"test.d.ts","sourceRoot":"","sources":["../../../src/modules/fetcher/test.ts"],"names":[],"mappings":""}

package/dist/modules/fetcher/test.js

@@ -0,0 +1,3 @@
+export {};
+// fetcher module - standalone test entry
+//# sourceMappingURL=test.js.map

package/dist/modules/fetcher/test.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"test.js","sourceRoot":"","sources":["../../../src/modules/fetcher/test.ts"],"names":[],"mappings":";AAAA,yCAAyC"}

package/dist/modules/flowus-source/index.d.ts

@@ -0,0 +1,55 @@
+/**
+ * flowus-source module
+ *
+ * FlowUs API client for reading articles from a multi-dimensional table.
+ * Replaces the wechaty bot as the article source in Phase 1.
+ */
+import type { FlowUsConfig, FlowUsPage, FlowUsBlock, FlowUsArticle } from './types.js';
+export declare class FlowUsClient {
+    private baseUrl;
+    private token;
+    private databaseId;
+    private headers;
+    constructor(config: FlowUsConfig);
+    /**
+     * Query all unprocessed articles from the database.
+     * Returns articles where "处理状态" is empty or "未处理".
+     */
+    queryNewArticles(): Promise<FlowUsPage[]>;
+    /**
+     * Read article content (all child blocks) and return as plain text/Markdown.
+     */
+    getArticleContent(pageId: string): Promise<string>;
+    /**
+     * Fetch all blocks for a page (raw, for inspection/testing).
+     */
+    getBlocks(pageId: string): Promise<FlowUsBlock[]>;
+    /**
+     * Mark an article as "处理中".
+     */
+    markProcessing(pageId: string): Promise<void>;
+    /**
+     * Mark an article as "已完成".
+     */
+    markAsProcessed(pageId: string): Promise<void>;
+    /**
+     * Mark an article as "失败".
+     */
+    markAsFailed(pageId: string, error?: string): Promise<void>;
+    private setStatus;
+    /**
+     * Convenience: get a full article with content from a page record.
+     */
+    getFullArticle(page: FlowUsPage): Promise<FlowUsArticle>;
+    /**
+     * Recursively fetch all blocks for a page, including nested children.
+     */
+    private fetchAllBlocks;
+    /**
+     * Convert a single block to Markdown string.
+     * FlowUs uses block.data (unlike Notion's block[block.type]).
+     */
+    private blockToMarkdown;
+}
+export type { FlowUsConfig, FlowUsPage, FlowUsBlock, FlowUsBlockData, FlowUsArticle } from './types.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/modules/flowus-source/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/modules/flowus-source/index.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAGH,OAAO,KAAK,EACV,YAAY,EACZ,UAAU,EACV,WAAW,EACX,aAAa,EAGd,MAAM,YAAY,CAAC;AAMpB,qBAAa,YAAY;IACvB,OAAO,CAAC,OAAO,CAAS;IACxB,OAAO,CAAC,KAAK,CAAS;IACtB,OAAO,CAAC,UAAU,CAAS;IAC3B,OAAO,CAAC,OAAO,CAAyB;gBAE5B,MAAM,EAAE,YAAY;IAUhC;;;OAGG;IACG,gBAAgB,IAAI,OAAO,CAAC,UAAU,EAAE,CAAC;IA+B/C;;OAEG;IACG,iBAAiB,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAKxD;;OAEG;IACG,SAAS,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC;IAIvD;;OAEG;IACG,cAAc,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAInD;;OAEG;IACG,eAAe,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAIpD;;OAEG;IACG,YAAY,CAAC,MAAM,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;YAInD,SAAS;IAgBvB;;OAEG;IACG,cAAc,CAAC,IAAI,EAAE,UAAU,GAAG,OAAO,CAAC,aAAa,CAAC;IAgC9D;;OAEG;YACW,cAAc;IA8B5B;;;OAGG;IACH,OAAO,CAAC,eAAe;CAuCxB;AAED,YAAY,EAAE,YAAY,EAAE,UAAU,EAAE,WAAW,EAAE,eAAe,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC"}