@cc-monitoring/proxy-core 0.2.3 → 0.3.1

package/dist/utils/__tests__/pi-runtime-json.test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/utils/__tests__/pi-runtime-json.test.js

@@ -0,0 +1,11 @@
+// pi-coding-agent 为 ESM-only（无 require 导出），jest CJS 无法加载；
+// mock 掉以加载真实 pi-runtime 模块测试纯函数（同 pi-browser-tools.test.ts 的 typebox mock 做法）
+jest.mock('@earendil-works/pi-coding-agent', () => ({ createWriteToolDefinition: () => ({}) }), { virtual: true });
+import { extractJsonObject } from '../pi-runtime.js';
+test('extractJsonObject: 从含 prose/代码围栏文本取出第一个 JSON 对象', () => {
+    const raw = 'Here:\n```json\n{"a":1,"b":[2,3]}\n```\ndone';
+    expect(extractJsonObject(raw)).toEqual({ a: 1, b: [2, 3] });
+});
+test('extractJsonObject: 无 JSON → throw', () => {
+    expect(() => extractJsonObject('no json here')).toThrow(/no json object/);
+});

package/dist/utils/__tests__/web-crawler.test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/utils/__tests__/web-crawler.test.js

@@ -0,0 +1,69 @@
+import { slugFromUrl, slugFromLabel, isSameSite, normalizeCandidate, } from '../web-crawler.js';
+describe('slugFromUrl', () => {
+    it('bare host without path', () => {
+        expect(slugFromUrl('https://pi.dev')).toBe('pi');
+    });
+    it('drops generic docs segment', () => {
+        expect(slugFromUrl('https://pi.dev/docs/intro')).toBe('pi-intro');
+    });
+    it('strips www and keeps at most 3 path segments', () => {
+        expect(slugFromUrl('https://www.example.com/a/b/c/d')).toBe('example-a-b-c');
+    });
+    it('ignores query and hash', () => {
+        expect(slugFromUrl('https://pi.dev/x?y=1#z')).toBe('pi-x');
+    });
+    it('throws on invalid url', () => {
+        expect(() => slugFromUrl('not a url')).toThrow();
+    });
+});
+describe('slugFromLabel', () => {
+    it('strips extension from ascii filename', () => {
+        expect(slugFromLabel('react-hooks.md')).toBe('react-hooks');
+    });
+    it('keeps the ascii stem and washes non-ascii chars away', () => {
+        expect(slugFromLabel('Pi 介绍.docx')).toBe('pi');
+    });
+    it('falls back to topic when nothing ascii remains', () => {
+        expect(slugFromLabel('介绍.docx')).toBe('topic');
+    });
+    it('takes basename and cleans spaces case-insensitively', () => {
+        expect(slugFromLabel('/tmp/dir/My Note.TXT')).toBe('my-note');
+    });
+});
+describe('isSameSite', () => {
+    it('subdomain is same site', () => {
+        expect(isSameSite('blog.pi.dev', 'pi.dev')).toBe(true);
+    });
+    it('www is equivalent to bare host', () => {
+        expect(isSameSite('www.pi.dev', 'pi.dev')).toBe(true);
+    });
+    it('cross domain is not same site', () => {
+        expect(isSameSite('other.com', 'pi.dev')).toBe(false);
+    });
+    it('suffix-without-dot is not same site', () => {
+        expect(isSameSite('notpi.dev', 'pi.dev')).toBe(false);
+    });
+});
+describe('normalizeCandidate', () => {
+    const entry = new URL('https://pi.dev/');
+    it('passes a normal http(s) link', () => {
+        const u = normalizeCandidate('https://pi.dev/a', entry);
+        expect(u?.href).toBe('https://pi.dev/a');
+    });
+    it('resolves relative links against entry', () => {
+        const u = normalizeCandidate('/about', entry);
+        expect(u?.href).toBe('https://pi.dev/about');
+    });
+    it('rejects non-http protocols like mailto', () => {
+        expect(normalizeCandidate('mailto:a@b.com', entry)).toBeNull();
+    });
+    it('excludes binary/document extensions', () => {
+        expect(normalizeCandidate('https://pi.dev/file.pdf', entry)).toBeNull();
+        expect(normalizeCandidate('https://pi.dev/PIC.JPG', entry)).toBeNull();
+    });
+    it('clears the hash', () => {
+        const u = normalizeCandidate('https://pi.dev/a#section', entry);
+        expect(u?.href).toBe('https://pi.dev/a');
+        expect(u?.hash).toBe('');
+    });
+});

package/dist/utils/db.d.ts

@@ -9,5 +9,6 @@ export declare function applySchemaV6(db: Database.Database): void;
 export declare function applySchemaV7(db: Database.Database): void;
 export declare function applySchemaV8(db: Database.Database): void;
 export declare function applySchemaV9(db: Database.Database): void;
+export declare function applySchemaV10(db: Database.Database): void;
 export declare function closeDatabase(): void;
 export declare function getDatabase(): Database.Database;

package/dist/utils/db.js

@@ -68,6 +68,10 @@ export function initDatabase() {
         applySchemaV9(db);
         db.prepare('INSERT OR REPLACE INTO schema_version (version) VALUES (9)').run();
     }
+    if (currentVersion < 10) {
+        applySchemaV10(db);
+        db.prepare('INSERT OR REPLACE INTO schema_version (version) VALUES (10)').run();
+    }
     return db;
 }
 export function applySchemaV2(db) {
@@ -146,6 +150,7 @@ function applySchemaV1(db) {
       proxy_api_key TEXT,
       base_url TEXT,
       model TEXT,
+      user_agent TEXT,
       input_tokens INTEGER,
       output_tokens INTEGER,
       cache_read_tokens INTEGER,
@@ -262,6 +267,19 @@ export function applySchemaV9(db) {
         db.exec("ALTER TABLE providers ADD COLUMN auth_style TEXT NOT NULL DEFAULT 'bearer'");
     }
 }
+export function applySchemaV10(db) {
+    // Add user_agent column to requests table
+    // ALTER TABLE ADD COLUMN is idempotent-safe via column-existence check
+    const columns = db.prepare("PRAGMA table_info(requests)").all();
+    if (!columns.some(c => c.name === 'user_agent')) {
+        db.exec('ALTER TABLE requests ADD COLUMN user_agent TEXT');
+    }
+    // Backfill historical rows from request_headers JSON (idempotent)
+    db.prepare(`
+    UPDATE requests SET user_agent = json_extract(request_headers, '$.user-agent')
+    WHERE user_agent IS NULL AND request_headers IS NOT NULL
+  `).run();
+}
 export function closeDatabase() {
     if (db) {
         db.close();

package/dist/utils/knowledge-store.d.ts

@@ -8,6 +8,7 @@ export interface ChunkRef {
     id: string;
     title: string;
     summary: string;
+    tags: string[];
 }
 export interface TopicDetail {
     title: string;
@@ -20,10 +21,17 @@ export interface ChunkHit {
     chunkId: string;
     title: string;
 }
+export interface QueryHit {
+    topic: string;
+    chunkId: string;
+    title: string;
+    body: string;
+}
 export interface KnowledgeStore {
     listTopics(): Promise<TopicSummary[]>;
     getTopic(topic: string): Promise<TopicDetail>;
     searchChunks(query: string): Promise<ChunkHit[]>;
+    queryKnowledge(query: string, limit?: number): Promise<QueryHit[]>;
     readChunk(topic: string, chunkId: string): Promise<string>;
     upsertTopic(input: {
         slug: string;
@@ -36,6 +44,7 @@ export interface KnowledgeStore {
         chunkId: string;
         title: string;
         summary: string;
+        tags: string[];
     }): Promise<void>;
 }
 export declare function createKnowledgeStore(root: string): KnowledgeStore;

package/dist/utils/knowledge-store.js

@@ -53,7 +53,14 @@ export function createKnowledgeStore(root) {
             title: meta.title || topic,
             tags: Array.isArray(meta.tags) ? meta.tags : [],
             sources: [],
-            chunks: Array.isArray(meta.chunks) ? meta.chunks : [],
+            chunks: Array.isArray(meta.chunks)
+                ? meta.chunks.map(c => ({
+                    id: c.id,
+                    title: c.title,
+                    summary: c.summary,
+                    tags: Array.isArray(c.tags) ? c.tags : [],
+                }))
+                : [],
         };
     }
     async function upsertTopic(input) {
@@ -90,7 +97,7 @@ export function createKnowledgeStore(root) {
     async function upsertChunkRef(input) {
         assertTopic(input.topic);
         assertChunkId(input.chunkId);
-        const { topic, chunkId, title, summary } = input;
+        const { topic, chunkId, title, summary, tags } = input;
         const topicPath = path.join(root, topic, 'index.json');
         let topicIdx;
         try {
@@ -109,17 +116,32 @@ export function createKnowledgeStore(root) {
         if (existing) {
             existing.title = title;
             existing.summary = summary;
+            existing.tags = tags;
         }
         else {
-            topicIdx.chunks.push({ id: chunkId, title, summary });
+            topicIdx.chunks.push({ id: chunkId, title, summary, tags });
         }
         await fs.writeFile(topicPath, JSON.stringify(topicIdx, null, 2), 'utf8');
     }
-    async function searchChunks(query) {
-        if (!query.trim())
-            return [];
-        const hits = [];
+    /** 匹配词集合：整体 query + whitespace 分割的长度 >1 的 token，去重 */
+    function buildTerms(query) {
         const q = query.toLowerCase();
+        const terms = new Set([q]);
+        for (const tok of q.split(/\s+/)) {
+            if (tok.length > 1)
+                terms.add(tok);
+        }
+        return [...terms];
+    }
+    /** 扫描所有 topic/chunk，返回打分候选（searchChunks / queryKnowledge 共用） */
+    async function scanChunks(query) {
+        const q = query.toLowerCase();
+        if (!q.trim())
+            return [];
+        const terms = buildTerms(query);
+        const candidates = [];
+        // topic index 本函数作用域缓存，避免每个 chunk 重复读
+        const topicIndexCache = new Map();
         let entries;
         try {
             entries = await fs.readdir(root);
@@ -130,6 +152,14 @@ export function createKnowledgeStore(root) {
         for (const topic of entries) {
             if (!TOPIC_RE.test(topic))
                 continue;
+            if (!topicIndexCache.has(topic)) {
+                const idx = await readJsonOr(path.join(root, topic, 'index.json'), null);
+                topicIndexCache.set(topic, idx);
+            }
+            const topicIdx = topicIndexCache.get(topic) ?? null;
+            const topicTags = topicIdx && Array.isArray(topicIdx.tags) ? topicIdx.tags : [];
+            const chunkRefs = topicIdx && Array.isArray(topicIdx.chunks) ? topicIdx.chunks : [];
+            const topicTagHit = (term) => topicTags.some(t => t.toLowerCase().includes(term));
             const chunksDir = path.join(root, topic, 'chunks');
             let files;
             try {
@@ -139,14 +169,58 @@ export function createKnowledgeStore(root) {
                 continue;
             }
             for (const file of files) {
-                if (!CHUNK_RE.test(file.replace(/\.md$/, '')))
+                const chunkId = file.replace(/\.md$/, '');
+                if (!CHUNK_RE.test(chunkId))
                     continue;
                 const text = await fs.readFile(path.join(chunksDir, file), 'utf8');
-                if (text.toLowerCase().includes(q)) {
-                    hits.push({ topic, chunkId: file.replace(/\.md$/, ''), title: parseChunkTitle(text) });
+                const textLc = text.toLowerCase();
+                const ref = chunkRefs.find(c => c.id === chunkId);
+                const refTitle = (ref?.title ?? '').toLowerCase();
+                const refSummary = (ref?.summary ?? '').toLowerCase();
+                const refTags = ref && Array.isArray(ref.tags) ? ref.tags : [];
+                let score = 0;
+                for (const term of terms) {
+                    // 每个字段每词只计一次分，body 不按出现次数刷分
+                    if (refTitle.includes(term))
+                        score += 3;
+                    if (refTags.some(t => t.toLowerCase().includes(term)))
+                        score += 2;
+                    if (topicTagHit(term))
+                        score += 2;
+                    if (refSummary.includes(term))
+                        score += 1;
+                    if (textLc.includes(term))
+                        score += 1;
                 }
+                if (score === 0)
+                    continue;
+                const wholeQueryHit = textLc.includes(q) ||
+                    refTitle.includes(q) ||
+                    refSummary.includes(q) ||
+                    topicTagHit(q) ||
+                    refTags.some(t => t.toLowerCase().includes(q));
+                candidates.push({ topic, chunkId, title: parseChunkTitle(text), score, wholeQueryHit });
             }
         }
+        return candidates;
+    }
+    async function searchChunks(query) {
+        if (!query.trim())
+            return [];
+        return (await scanChunks(query))
+            .filter(c => c.wholeQueryHit)
+            .map(({ topic, chunkId, title }) => ({ topic, chunkId, title }));
+    }
+    async function queryKnowledge(query, limit = 3) {
+        const capped = Math.min(Math.max(limit, 1), 5);
+        const candidates = (await scanChunks(query))
+            .sort((a, b) => b.score - a.score)
+            .slice(0, capped);
+        const hits = [];
+        for (const c of candidates) {
+            const body = await readChunk(c.topic, c.chunkId);
+            hits.push({ topic: c.topic, chunkId: c.chunkId, title: c.title, body });
+        }
         return hits;
     }
     async function readChunk(topic, chunkId) {
@@ -161,7 +235,7 @@ export function createKnowledgeStore(root) {
         }
         return fs.readFile(resolved, 'utf8');
     }
-    return { listTopics, getTopic, searchChunks, readChunk, upsertTopic, upsertChunkRef };
+    return { listTopics, getTopic, searchChunks, queryKnowledge, readChunk, upsertTopic, upsertChunkRef };
 }
 // 默认单例：绑生产 docsDir
 export const knowledgeStore = createKnowledgeStore(docsDir);

package/dist/utils/pi-browser-tools.d.ts

@@ -7,4 +7,6 @@ export declare function runEgoBrowser(scriptBody: string, opts?: {
     timeoutMs?: number;
     signal?: AbortSignal;
 }): Promise<string>;
+/** 过滤 ego-browser 自身打到输出流的横幅行（如升级 notice），保留脚本 cliLog 的真实输出 */
+export declare function stripCliBanners(text: string): string;
 export declare const browserTools: ToolDefinition[];

package/dist/utils/pi-browser-tools.js

@@ -8,7 +8,7 @@ export function runEgoBrowser(scriptBody, opts = {}) {
     const timeoutMs = opts.timeoutMs ?? 30000;
     return new Promise((resolve, reject) => {
         // 头部自动注入 task space 复用调用
-        const script = `const task = await useOrCreateTaskSpace('knowledge-learn')\n${scriptBody}`;
+        const script = `const task = await taskSpace('knowledge-learn')\n${scriptBody}`;
         const child = spawn('ego-browser', ['nodejs'], {
             stdio: ['pipe', 'pipe', 'pipe'],
         });
@@ -61,7 +61,7 @@ export function runEgoBrowser(scriptBody, opts = {}) {
                 return;
             if (code === 0) {
                 // 新版 ego cliLog 走 stderr；stdout 非空用 stdout，否则回退 stderr（新旧兼容）
-                const out = stdout.trim() ? stdout : stderr;
+                const out = stripCliBanners(stdout.trim() ? stdout : stderr);
                 finish(null, out);
             }
             else {
@@ -73,6 +73,13 @@ export function runEgoBrowser(scriptBody, opts = {}) {
         child.stdin.end(script, 'utf-8');
     });
 }
+/** 过滤 ego-browser 自身打到输出流的横幅行（如升级 notice），保留脚本 cliLog 的真实输出 */
+export function stripCliBanners(text) {
+    return text
+        .split('\n')
+        .filter((line) => !line.startsWith('[ego-browser:notice]'))
+        .join('\n');
+}
 // ---------- 工具实现辅助 ----------
 /** 把字符串参数安全注入脚本：JSON.stringify 自动处理引号 / 反斜杠 / 换行 */
 const jsLit = (v) => JSON.stringify(v);

package/dist/utils/pi-crawl-tools.d.ts

@@ -0,0 +1,62 @@
+import type { ToolDefinition } from '@earendil-works/pi-coding-agent';
+export interface NavigatedPage {
+    title: string;
+    text: string;
+    links: string[];
+}
+interface PageData extends NavigatedPage {
+    url: string;
+}
+export interface CrawlGuardOptions {
+    entryUrl: string;
+    docsDir: string;
+    slug: string;
+    maxPages?: number;
+    maxDepth?: number;
+    /** 可注入的单页抓取（测试用）；默认调 runEgoBrowser；signal 用于中断在途抓取 */
+    navigate?: (url: string, signal?: AbortSignal) => Promise<NavigatedPage>;
+}
+export interface SaveSourceResult {
+    savedCount: number;
+    reused: boolean;
+}
+export declare class CrawlGuard {
+    private readonly opts;
+    private readonly maxPages;
+    private readonly maxDepth;
+    private readonly navigate;
+    private entry;
+    private readonly seen;
+    private readonly depthByUrl;
+    private readonly rawCache;
+    private readonly savedUrls;
+    private _savedCount;
+    private currentDepth;
+    constructor(opts: CrawlGuardOptions);
+    /** 默认单页抓取：跑 ego 脚本，JSON parse 失败硬抛错；signal 透传中断在途进程 */
+    private defaultNavigate;
+    /** 打开入口页：URL 校验 + 初始状态；signal 用于中断在途抓取 */
+    open(signal?: AbortSignal): Promise<PageData>;
+    /**
+     * 解析候选页的 BFS 最短深度：扫描每个已抓来源页的 links，
+     * 凡归一后命中候选页的来源，候选深度 = 来源深度 + 1，取所有命中的最小值。
+     * 一个来源都没查到（幻觉 URL）时硬抛错，绝不放行。
+     */
+    private resolveDepth;
+    /** 访问子页：同站/seen/链接来源/深度/页数硬校验（原因 cross-site/seen/not linked/max-depth/max-pages） */
+    visit(rawUrl: string, signal?: AbortSignal): Promise<PageData>;
+    /**
+     * 保存已抓取页原文到 <slug>/sources/source-NNN.md；必须先 open/visit 过该 URL。
+     * 同页重复保存不写新文件，返回已有编号 + reused=true；空正文页判失败。
+     */
+    saveSource(rawUrl: string): SaveSourceResult;
+    get savedCount(): number;
+    /**
+     * 抓取结束清理：no-op。
+     * 不能调 task.finish()——它会关闭全部 page 并把 space 所有权交还 user，
+     * 导致下一次 learn 的 taskSpace() resume 失败；持久 space 与 tab 留给后续复用。
+     */
+    close(): Promise<void>;
+}
+export declare function createCrawlTools(guard: CrawlGuard): ToolDefinition[];
+export {};