@massa-ai/core 1.2.1

package/dist/services/synapse/types.d.ts

@@ -0,0 +1,79 @@
+/**
+ * Synapse — shared types for the cognitive modulation layer.
+ *
+ * Synapse is an interceptor layer over the existing retrieval pipeline.
+ * It does not replace search; it modulates which results survive and in what order.
+ */
+import type { SearchResult } from "@massa-ai/shared";
+import type { WorkingMemoryBuffer } from "./buffer/working-memory-buffer.js";
+/**
+ * Classification of a query, used by the adaptive confidence gate.
+ */
+export type QueryClass = "specific" | "focused" | "broad";
+/**
+ * Metacognition flags surfaced to callers when Synapse analyzes its own confidence.
+ */
+export interface SpectrumFlags {
+    lowConfidence: boolean;
+    noStrongMatch: boolean;
+    definitiveMatch: boolean;
+    spread: number;
+    mean: number;
+    confidence: number;
+}
+/**
+ * Result of running the full post-retrieval Synapse pipeline.
+ */
+export interface SynapsePipelineResult {
+    results: SearchResult[];
+    flags: SpectrumFlags;
+    queryClass: QueryClass;
+    appliedFilters: string[];
+    intent: QueryIntent;
+}
+/**
+ * Agent session — ephemeral context for a single agent working on a task.
+ * Kept in-memory; persistence is intentionally out of scope here.
+ */
+export interface AgentSession {
+    sessionId: string;
+    agentId: string;
+    workspaceId?: string;
+    taskContext?: string;
+    taskTokens?: Set<string>;
+    taskEmbedding?: Float32Array | number[];
+    ttlMs: number;
+    createdAt: number;
+    expiresAt: number;
+    /**
+     * memoryId -> number of accesses by this agent inside the session.
+     *
+     * The Map is bounded — when it exceeds `accessHistoryLimit`, the LEAST
+     * RECENTLY recorded entry is evicted (IMP-11). This relies on the JS Map's
+     * insertion-order iteration: re-recording an existing key requires a
+     * delete-then-set to refresh recency.
+     */
+    accessHistory: Map<string, number>;
+    /** Hard cap on accessHistory.size; new entries past this evict the oldest. */
+    accessHistoryLimit: number;
+    /** Optional working-memory buffer; populated when buffer is enabled in config. */
+    buffer?: WorkingMemoryBuffer;
+}
+/**
+ * Mapping of detected query intent to memory chain weights.
+ * Values multiply the score of a result whose `metadata.type` matches the chain.
+ */
+export type QueryIntent = "decision" | "debug" | "pattern" | "symbol" | "general";
+export interface ChainBoostMap {
+    decision?: number;
+    code?: number;
+    /** Synthetic chain inferred from filePath when source is a test file. */
+    "code-test"?: number;
+    pattern?: number;
+    conversation?: number;
+    preference?: number;
+    critical?: number;
+    /** Synthetic chain inferred from filePath when source is a doc/README. */
+    documentation?: number;
+}
+//# sourceMappingURL=types.d.ts.map

package/dist/services/synapse/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../../src/services/synapse/types.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;AACrD,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,mCAAmC,CAAC;AAE7E;;GAEG;AACH,MAAM,MAAM,UAAU,GAAG,UAAU,GAAG,SAAS,GAAG,OAAO,CAAC;AAE1D;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,aAAa,EAAE,OAAO,CAAC;IACvB,aAAa,EAAE,OAAO,CAAC;IACvB,eAAe,EAAE,OAAO,CAAC;IACzB,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,EAAE,MAAM,CAAC;CACpB;AAED;;GAEG;AACH,MAAM,WAAW,qBAAqB;IACpC,OAAO,EAAE,YAAY,EAAE,CAAC;IACxB,KAAK,EAAE,aAAa,CAAC;IACrB,UAAU,EAAE,UAAU,CAAC;IACvB,cAAc,EAAE,MAAM,EAAE,CAAC;IACzB,MAAM,EAAE,WAAW,CAAC;CACrB;AAED;;;GAGG;AACH,MAAM,WAAW,YAAY;IAC3B,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,UAAU,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;IACzB,aAAa,CAAC,EAAE,YAAY,GAAG,MAAM,EAAE,CAAC;IACxC,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB;;;;;;;OAOG;IACH,aAAa,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACnC,8EAA8E;IAC9E,kBAAkB,EAAE,MAAM,CAAC;IAC3B,kFAAkF;IAClF,MAAM,CAAC,EAAE,mBAAmB,CAAC;CAC9B;AAED;;;GAGG;AACH,MAAM,MAAM,WAAW,GACnB,UAAU,GACV,OAAO,GACP,SAAS,GACT,QAAQ,GACR,SAAS,CAAC;AAEd,MAAM,WAAW,aAAa;IAC5B,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,yEAAyE;IACzE,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,0EAA0E;IAC1E,aAAa,CAAC,EAAE,MAAM,CAAC;CACxB"}

package/dist/services/synapse/types.js

@@ -0,0 +1,8 @@
+/**
+ * Synapse — shared types for the cognitive modulation layer.
+ *
+ * Synapse is an interceptor layer over the existing retrieval pipeline.
+ * It does not replace search; it modulates which results survive and in what order.
+ */
+export {};
+//# sourceMappingURL=types.js.map

package/dist/services/synapse/types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../../../src/services/synapse/types.ts"],"names":[],"mappings":"AAAA;;;;;GAKG"}

package/dist/services/web/fetcher.d.ts

@@ -0,0 +1,99 @@
+/**
+ * Web fetcher + indexer for `fetch_and_index`.
+ *
+ * Flow (single URL):
+ *   1. SSRF guard: assertUrlSafe resolves + classifies the hostname's IPs.
+ *   2. TTL cache check: if a fresh entry exists for this (source, url) key,
+ *      return a cache-hit result WITHOUT fetching or indexing.
+ *   3. fetchWithSsrfGuard: manual redirect walk — each hop re-resolved + classified.
+ *   4. Content-Length + read byte cap (MAX_FETCH_BYTES) — reject oversized before
+ *      slurping the body into the long-running process heap.
+ *   5. Convert: HTML → markdown (turndown+gfm), JSON → key-path chunks,
+ *      everything else → plain text.
+ *   6. Index SERIALY via the injected `indexChunk` seam (single-writer safety —
+ *      the underlying PostgreSQL FTS5 / pgvector stores are not safe under
+ *      concurrent writers, and even pgvector embeddings serialize better one
+ *      doc at a time). The PARALLEL phase is the fetch (run-pool in the
+ *      controller); the SERIAL phase is always this drain.
+ *
+ * The indexing layer is injected (`WebIndexDeps`) so unit tests can capture
+ * indexed chunks against an in-memory map instead of the real vector store.
+ */
+import { type FetchGuardOptions } from "./ssrf.js";
+/** Hard cap on fetched body size (50 MB). Checked against Content-Length AND
+ *  the actual bytes read (defense-in-depth: a lying Content-Length or chunked
+ *  transfer can still slip past the header check). */
+export declare const MAX_FETCH_BYTES: number;
+/** Default TTL for the fetch cache (24h). */
+export declare const DEFAULT_FETCH_TTL_MS: number;
+export type FetchContentType = "html" | "json" | "text";
+/** A single indexed chunk produced from a fetch. */
+export interface IndexedChunk {
+    /** Stable id: projectId:url-hash:idx — unique per (url, position). */
+    id: string;
+    content: string;
+    metadata: Record<string, unknown>;
+}
+/** Per-fetch result returned to the controller. */
+export type FetchOneResult = {
+    kind: "cached";
+    url: string;
+    source: string;
+    chunkCount: number;
+    ageMs: number;
+} | {
+    kind: "fetched";
+    url: string;
+    source: string;
+    contentType: FetchContentType;
+    chunks: IndexedChunk[];
+    bytes: number;
+} | {
+    kind: "error";
+    url: string;
+    error: string;
+};
+/**
+ * Indexing seams injected by the controller. Keeping these as an interface (not
+ * a hard dependency on the vector/keyword stores) lets tests assert indexed
+ * chunks without a DB, and lets the controller compose vector+keyword writes.
+ */
+export interface WebIndexDeps {
+    /**
+     * Index one chunk into BOTH stores. Called SERIALY by the fetcher (caller
+     * awaits each call). Implementations typically do:
+     *   await Promise.all([
+     *     vectorStore.addDocuments([doc]),
+     *     keywordSearch.index(doc.id, doc.content, doc.metadata),
+     *   ]);
+     */
+    indexChunk(chunk: IndexedChunk): Promise<void>;
+    /**
+     * Return the timestamp (ms since epoch) this (source,url) was last indexed,
+     * or `null` if unknown / stale-beyond-repair. Used for TTL cache hits.
+     */
+    getLastIndexedAt?(cacheKey: string): number | null;
+    /** Record that indexing just finished for this cacheKey (post-write). */
+    markIndexed?(cacheKey: string, ts: number): void;
+}
+/** Hash the (source, url) tuple into a stable cache/storage key. */
+export declare function composeFetchCacheKey(source: string | undefined, url: string): string;
+export interface FetchOneOptions extends FetchGuardOptions {
+    source?: string;
+    /** Skip cache and re-fetch. */
+    force?: boolean;
+    /** Override cache window for this call (ms). `0` bypasses cache like force. */
+    ttl?: number;
+    /** projectId scoping the indexed chunks. Defaults to a synthetic web pid. */
+    projectId?: string;
+}
+/**
+ * Fetch + convert + index a single URL. Safe to run in PARALLEL on the fetch
+ * phase; the indexing drain inside is SERIAL (awaits indexChunk per chunk).
+ *
+ * `projectId` defaults to `"web"` so fetched content lives in its own scope
+ * unless the caller namespaces it. Tests should pass a throwaway pid to avoid
+ * polluting any shared index.
+ */
+export declare function fetchAndConvertOne(url: string, deps: WebIndexDeps, opts?: FetchOneOptions): Promise<FetchOneResult>;
+//# sourceMappingURL=fetcher.d.ts.map

package/dist/services/web/fetcher.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"fetcher.d.ts","sourceRoot":"","sources":["../../../src/services/web/fetcher.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;GAoBG;AAIH,OAAO,EAGL,KAAK,iBAAiB,EACvB,MAAM,WAAW,CAAC;AAGnB;;sDAEsD;AACtD,eAAO,MAAM,eAAe,QAAmB,CAAC;AAEhD,6CAA6C;AAC7C,eAAO,MAAM,oBAAoB,QAAsB,CAAC;AAExD,MAAM,MAAM,gBAAgB,GAAG,MAAM,GAAG,MAAM,GAAG,MAAM,CAAC;AAExD,oDAAoD;AACpD,MAAM,WAAW,YAAY;IAC3B,sEAAsE;IACtE,EAAE,EAAE,MAAM,CAAC;IACX,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACnC;AAED,mDAAmD;AACnD,MAAM,MAAM,cAAc,GACtB;IACE,IAAI,EAAE,QAAQ,CAAC;IACf,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,EAAE,MAAM,CAAC;IACnB,KAAK,EAAE,MAAM,CAAC;CACf,GACD;IACE,IAAI,EAAE,SAAS,CAAC;IAChB,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,EAAE,MAAM,CAAC;IACf,WAAW,EAAE,gBAAgB,CAAC;IAC9B,MAAM,EAAE,YAAY,EAAE,CAAC;IACvB,KAAK,EAAE,MAAM,CAAC;CACf,GACD;IACE,IAAI,EAAE,OAAO,CAAC;IACd,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,MAAM,CAAC;CACf,CAAC;AAEN;;;;GAIG;AACH,MAAM,WAAW,YAAY;IAC3B;;;;;;;OAOG;IACH,UAAU,CAAC,KAAK,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAC/C;;;OAGG;IACH,gBAAgB,CAAC,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAAC;IACnD,yEAAyE;IACzE,WAAW,CAAC,CAAC,QAAQ,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,GAAG,IAAI,CAAC;CAClD;AAED,oEAAoE;AACpE,wBAAgB,oBAAoB,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,EAAE,GAAG,EAAE,MAAM,GAAG,MAAM,CAIpF;AA0DD,MAAM,WAAW,eAAgB,SAAQ,iBAAiB;IACxD,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,+BAA+B;IAC/B,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,+EAA+E;IAC/E,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,6EAA6E;IAC7E,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED;;;;;;;GAOG;AACH,wBAAsB,kBAAkB,CACtC,GAAG,EAAE,MAAM,EACX,IAAI,EAAE,YAAY,EAClB,IAAI,GAAE,eAAoB,GACzB,OAAO,CAAC,cAAc,CAAC,CA8FzB"}

package/dist/services/web/fetcher.js

@@ -0,0 +1,267 @@
+/**
+ * Web fetcher + indexer for `fetch_and_index`.
+ *
+ * Flow (single URL):
+ *   1. SSRF guard: assertUrlSafe resolves + classifies the hostname's IPs.
+ *   2. TTL cache check: if a fresh entry exists for this (source, url) key,
+ *      return a cache-hit result WITHOUT fetching or indexing.
+ *   3. fetchWithSsrfGuard: manual redirect walk — each hop re-resolved + classified.
+ *   4. Content-Length + read byte cap (MAX_FETCH_BYTES) — reject oversized before
+ *      slurping the body into the long-running process heap.
+ *   5. Convert: HTML → markdown (turndown+gfm), JSON → key-path chunks,
+ *      everything else → plain text.
+ *   6. Index SERIALY via the injected `indexChunk` seam (single-writer safety —
+ *      the underlying PostgreSQL FTS5 / pgvector stores are not safe under
+ *      concurrent writers, and even pgvector embeddings serialize better one
+ *      doc at a time). The PARALLEL phase is the fetch (run-pool in the
+ *      controller); the SERIAL phase is always this drain.
+ *
+ * The indexing layer is injected (`WebIndexDeps`) so unit tests can capture
+ * indexed chunks against an in-memory map instead of the real vector store.
+ */
+import { createHash } from "node:crypto";
+import { logger } from "@massa-ai/shared";
+import { fetchWithSsrfGuard, SsrfBlockedError, } from "./ssrf.js";
+import { htmlToMarkdown, jsonToKeyPathChunks } from "./html-to-md.js";
+/** Hard cap on fetched body size (50 MB). Checked against Content-Length AND
+ *  the actual bytes read (defense-in-depth: a lying Content-Length or chunked
+ *  transfer can still slip past the header check). */
+export const MAX_FETCH_BYTES = 50 * 1024 * 1024;
+/** Default TTL for the fetch cache (24h). */
+export const DEFAULT_FETCH_TTL_MS = 24 * 60 * 60 * 1000;
+/** Hash the (source, url) tuple into a stable cache/storage key. */
+export function composeFetchCacheKey(source, url) {
+    const label = source && source.trim() ? source.trim() : url;
+    const h = createHash("sha256").update(url).digest("hex").slice(0, 16);
+    return `${label}#${h}`;
+}
+/** Internal: what content-type bucket does a response fall into? */
+function classifyContentType(contentTypeHeader) {
+    const ct = contentTypeHeader.toLowerCase();
+    if (ct.includes("application/json") || ct.includes("+json"))
+        return "json";
+    if (ct.includes("text/html") || ct.includes("application/xhtml"))
+        return "html";
+    return "text";
+}
+/**
+ * Read the response body with a hard byte ceiling. Checks Content-Length first
+ * (cheap, before any buffering) and then caps the actual streamed read so a
+ * lying header or chunked encoding can't exhaust memory.
+ */
+async function readCappedBody(resp) {
+    const cl = parseInt(resp.headers.get("content-length") || "0", 10);
+    if (cl && cl > MAX_FETCH_BYTES) {
+        throw new Error(`Content-Length ${cl} exceeds cap ${MAX_FETCH_BYTES}`);
+    }
+    // Read in chunks so we can abort mid-stream if the body exceeds the cap.
+    const reader = resp.body?.getReader();
+    if (!reader) {
+        // No streaming body (e.g. some redirect responses already drained) — fall
+        // back to .text(). Already bounded by Content-Length above when present.
+        const text = await resp.text();
+        if (text.length > MAX_FETCH_BYTES) {
+            throw new Error(`body ${text.length} bytes exceeds cap ${MAX_FETCH_BYTES}`);
+        }
+        return text;
+    }
+    const decoder = new TextDecoder("utf-8");
+    let received = 0;
+    let out = "";
+    while (true) {
+        const { done, value } = await reader.read();
+        if (done)
+            break;
+        received += value.byteLength;
+        if (received > MAX_FETCH_BYTES) {
+            try {
+                await reader.cancel();
+            }
+            catch {
+                /* ignore */
+            }
+            throw new Error(`streamed body exceeded cap ${MAX_FETCH_BYTES} (read ${received} bytes)`);
+        }
+        out += decoder.decode(value, { stream: true });
+    }
+    out += decoder.decode(); // flush
+    return out;
+}
+/**
+ * Fetch + convert + index a single URL. Safe to run in PARALLEL on the fetch
+ * phase; the indexing drain inside is SERIAL (awaits indexChunk per chunk).
+ *
+ * `projectId` defaults to `"web"` so fetched content lives in its own scope
+ * unless the caller namespaces it. Tests should pass a throwaway pid to avoid
+ * polluting any shared index.
+ */
+export async function fetchAndConvertOne(url, deps, opts = {}) {
+    const { source, force, ttl, projectId = "web", timeoutMs, signal, _compose } = opts;
+    const cacheKey = composeFetchCacheKey(source, url);
+    // TTL cache hit — skip both fetch and index. `force` or `ttl: 0` bypass.
+    if (!force && ttl !== 0 && deps.getLastIndexedAt) {
+        const last = deps.getLastIndexedAt(cacheKey);
+        if (last !== null && last !== undefined) {
+            const ageMs = Date.now() - last;
+            const windowMs = ttl ?? DEFAULT_FETCH_TTL_MS;
+            if (ageMs < windowMs) {
+                // chunkCount 0 in the public result: the chunks are NOT re-returned on a
+                // cache hit (they were indexed on the original fetch). The earlier
+                // internal sentinel (-1) leaked into tool output; 0 reads correctly to
+                // callers as "no chunks in this response."
+                return { kind: "cached", url, source: source || url, chunkCount: 0, ageMs };
+            }
+        }
+    }
+    let resp;
+    try {
+        resp = await fetchWithSsrfGuard(url, { timeoutMs, signal, _compose });
+    }
+    catch (err) {
+        if (err instanceof SsrfBlockedError) {
+            return { kind: "error", url, error: err.message };
+        }
+        const msg = err instanceof Error ? err.message : String(err);
+        logger.warn?.("fetch_and_index fetch failed", { url, error: msg });
+        return { kind: "error", url, error: `fetch failed: ${msg}` };
+    }
+    if (!resp.ok) {
+        await resp.body?.cancel().catch(() => { });
+        return { kind: "error", url, error: `HTTP ${resp.status} ${resp.statusText}` };
+    }
+    const contentType = classifyContentType(resp.headers.get("content-type") || "");
+    let bodyText;
+    try {
+        bodyText = await readCappedBody(resp);
+    }
+    catch (err) {
+        const msg = err instanceof Error ? err.message : String(err);
+        return { kind: "error", url, error: msg };
+    }
+    finally {
+        await resp.body?.cancel().catch(() => { });
+    }
+    if (!bodyText.trim()) {
+        return { kind: "error", url, error: "empty body" };
+    }
+    // Build the chunk set based on content type.
+    const chunks = buildChunks(bodyText, contentType, url, source, projectId, cacheKey);
+    // SERIAL index drain — single writer. Fetches raced in the controller's
+    // run-pool; this drain is the part that must not parallelize.
+    for (const chunk of chunks) {
+        try {
+            await deps.indexChunk(chunk);
+        }
+        catch (err) {
+            const msg = err instanceof Error ? err.message : String(err);
+            logger.error("fetch_and_index indexChunk failed", err, {
+                url,
+                chunkId: chunk.id,
+            });
+            return {
+                kind: "error",
+                url,
+                error: `indexing failed at chunk ${chunk.id}: ${msg}`,
+            };
+        }
+    }
+    deps.markIndexed?.(cacheKey, Date.now());
+    return {
+        kind: "fetched",
+        url,
+        source: source || url,
+        contentType,
+        chunks,
+        bytes: Buffer.byteLength(bodyText, "utf-8"),
+    };
+}
+/**
+ * Convert a fetched body into IndexedChunk[] based on its content-type bucket.
+ * HTML → markdown (then one chunk per ~chunk-size slice); JSON → one chunk per
+ * leaf key-path; text → a single chunk. Every chunk's metadata carries the
+ * source url, cacheKey, and projectId so `search` can filter them back out.
+ */
+function buildChunks(body, contentType, url, source, projectId, cacheKey) {
+    const baseMeta = {
+        projectId,
+        source: source || url,
+        url,
+        cacheKey,
+        fetchedAt: new Date().toISOString(),
+    };
+    const urlHash = createHash("sha256").update(url).digest("hex").slice(0, 12);
+    if (contentType === "json") {
+        let parsed = body;
+        try {
+            parsed = JSON.parse(body);
+        }
+        catch {
+            // Not valid JSON despite the header — treat as text.
+            parsed = body;
+        }
+        if (typeof parsed === "string" || typeof parsed === "number" || typeof parsed === "boolean") {
+            return [
+                {
+                    id: `${projectId}:web:${urlHash}:0`,
+                    content: `**$** = \`${String(parsed)}\`\n\n_source: ${url}_`,
+                    metadata: { ...baseMeta, type: "json_leaf", label: "$" },
+                },
+            ];
+        }
+        const jc = jsonToKeyPathChunks(parsed);
+        if (jc.length === 0) {
+            return [
+                {
+                    id: `${projectId}:web:${urlHash}:0`,
+                    content: body.slice(0, 7500),
+                    metadata: { ...baseMeta, type: "json_text" },
+                },
+            ];
+        }
+        return jc.map((c, i) => ({
+            id: `${projectId}:web:${urlHash}:${i}`,
+            content: `${c.content}\n\n_source: ${url}_`,
+            metadata: { ...baseMeta, type: "json_key", label: c.path, path: c.path },
+        }));
+    }
+    if (contentType === "html") {
+        const md = htmlToMarkdown(body);
+        if (!md)
+            return [];
+        // Slice oversized markdown into ~7500-char chunks (matches the chunker's
+        // maxChunkChars cap) so no single doc overflows the embedding model.
+        return sliceMarkdown(md, 7500).map((slice, i) => ({
+            id: `${projectId}:web:${urlHash}:${i}`,
+            content: `_source: ${url}_\n\n${slice}`,
+            metadata: { ...baseMeta, type: "web_html", chunkIndex: i },
+        }));
+    }
+    // Plain text / CSV / XML / etc.
+    return [
+        {
+            id: `${projectId}:web:${urlHash}:0`,
+            content: `_source: ${url}_\n\n${body.slice(0, 7500)}`,
+            metadata: { ...baseMeta, type: "web_text" },
+        },
+    ];
+}
+/** Split markdown on paragraph boundaries near every `size` chars. */
+function sliceMarkdown(md, size) {
+    if (md.length <= size)
+        return [md];
+    const out = [];
+    let start = 0;
+    while (start < md.length) {
+        let end = Math.min(start + size, md.length);
+        if (end < md.length) {
+            // Try to break on a blank line within the last 500 chars of the window.
+            const breakRegion = md.lastIndexOf("\n\n", end);
+            if (breakRegion > start + size * 0.5)
+                end = breakRegion + 2;
+        }
+        out.push(md.slice(start, end));
+        start = end;
+    }
+    return out.filter((s) => s.trim().length > 0);
+}
+//# sourceMappingURL=fetcher.js.map

package/dist/services/web/fetcher.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"fetcher.js","sourceRoot":"","sources":["../../../src/services/web/fetcher.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;GAoBG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAC1C,OAAO,EACL,kBAAkB,EAClB,gBAAgB,GAEjB,MAAM,WAAW,CAAC;AACnB,OAAO,EAAE,cAAc,EAAE,mBAAmB,EAAE,MAAM,iBAAiB,CAAC;AAEtE;;sDAEsD;AACtD,MAAM,CAAC,MAAM,eAAe,GAAG,EAAE,GAAG,IAAI,GAAG,IAAI,CAAC;AAEhD,6CAA6C;AAC7C,MAAM,CAAC,MAAM,oBAAoB,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC;AA2DxD,oEAAoE;AACpE,MAAM,UAAU,oBAAoB,CAAC,MAA0B,EAAE,GAAW;IAC1E,MAAM,KAAK,GAAG,MAAM,IAAI,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC;IAC5D,MAAM,CAAC,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IACtE,OAAO,GAAG,KAAK,IAAI,CAAC,EAAE,CAAC;AACzB,CAAC;AAED,oEAAoE;AACpE,SAAS,mBAAmB,CAC1B,iBAAyB;IAEzB,MAAM,EAAE,GAAG,iBAAiB,CAAC,WAAW,EAAE,CAAC;IAC3C,IAAI,EAAE,CAAC,QAAQ,CAAC,kBAAkB,CAAC,IAAI,EAAE,CAAC,QAAQ,CAAC,OAAO,CAAC;QAAE,OAAO,MAAM,CAAC;IAC3E,IAAI,EAAE,CAAC,QAAQ,CAAC,WAAW,CAAC,IAAI,EAAE,CAAC,QAAQ,CAAC,mBAAmB,CAAC;QAAE,OAAO,MAAM,CAAC;IAChF,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;;;GAIG;AACH,KAAK,UAAU,cAAc,CAAC,IAAc;IAC1C,MAAM,EAAE,GAAG,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAC,IAAI,GAAG,EAAE,EAAE,CAAC,CAAC;IACnE,IAAI,EAAE,IAAI,EAAE,GAAG,eAAe,EAAE,CAAC;QAC/B,MAAM,IAAI,KAAK,CACb,kBAAkB,EAAE,gBAAgB,eAAe,EAAE,CACtD,CAAC;IACJ,CAAC;IACD,yEAAyE;IACzE,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,EAAE,SAAS,EAAE,CAAC;IACtC,IAAI,CAAC,MAAM,EAAE,CAAC;QACZ,0EAA0E;QAC1E,yEAAyE;QACzE,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;QAC/B,IAAI,IAAI,CAAC,MAAM,GAAG,eAAe,EAAE,CAAC;YAClC,MAAM,IAAI,KAAK,CAAC,QAAQ,IAAI,CAAC,MAAM,sBAAsB,eAAe,EAAE,CAAC,CAAC;QAC9E,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC;IACD,MAAM,OAAO,GAAG,IAAI,WAAW,CAAC,OAAO,CAAC,CAAC;IACzC,IAAI,QAAQ,GAAG,CAAC,CAAC;IACjB,IAAI,GAAG,GAAG,EAAE,CAAC;IACb,OAAO,IAAI,EAAE,CAAC;QACZ,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;QAC5C,IAAI,IAAI;YAAE,MAAM;QAChB,QAAQ,IAAI,KAAK,CAAC,UAAU,CAAC;QAC7B,IAAI,QAAQ,GAAG,eAAe,EAAE,CAAC;YAC/B,IAAI,CAAC;gBACH,MAAM,MAAM,CAAC,MAAM,EAAE,CAAC;YACxB,CAAC;YAAC,MAAM,CAAC;gBACP,YAAY;YACd,CAAC;YACD,MAAM,IAAI,KAAK,CACb,8BAA8B,eAAe,UAAU,QAAQ,SAAS,CACzE,CAAC;QACJ,CAAC;QACD,GAAG,IAAI,OAAO,CAAC,MAAM,CAAC,KAAK,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC;IACjD,CAAC;IACD,GAAG,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ;IACjC,OAAO,GAAG,CAAC;AACb,CAAC;AAYD;;;;;;;GAOG;AACH,MAAM,CAAC,KAAK,UAAU,kBAAkB,CACtC,GAAW,EACX,IAAkB,EAClB,OAAwB,EAAE;IAE1B,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,GAAG,EAAE,SAAS,GAAG,KAAK,EAAE,SAAS,EAAE,MAAM,EAAE,QAAQ,EAAE,GAAG,IAAI,CAAC;IACpF,MAAM,QAAQ,GAAG,oBAAoB,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;IAEnD,yEAAyE;IACzE,IAAI,CAAC,KAAK,IAAI,GAAG,KAAK,CAAC,IAAI,IAAI,CAAC,gBAAgB,EAAE,CAAC;QACjD,MAAM,IAAI,GAAG,IAAI,CAAC,gBAAgB,CAAC,QAAQ,CAAC,CAAC;QAC7C,IAAI,IAAI,KAAK,IAAI,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;YACxC,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC;YAChC,MAAM,QAAQ,GAAG,GAAG,IAAI,oBAAoB,CAAC;YAC7C,IAAI,KAAK,GAAG,QAAQ,EAAE,CAAC;gBACrB,yEAAyE;gBACzE,mEAAmE;gBACnE,uEAAuE;gBACvE,2CAA2C;gBAC3C,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,GAAG,EAAE,MAAM,EAAE,MAAM,IAAI,GAAG,EAAE,UAAU,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC;YAC9E,CAAC;QACH,CAAC;IACH,CAAC;IAED,IAAI,IAAc,CAAC;IACnB,IAAI,CAAC;QACH,IAAI,GAAG,MAAM,kBAAkB,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC,CAAC;IACxE,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,IAAI,GAAG,YAAY,gBAAgB,EAAE,CAAC;YACpC,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,GAAG,EAAE,KAAK,EAAE,GAAG,CAAC,OAAO,EAAE,CAAC;QACpD,CAAC;QACD,MAAM,GAAG,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QAC7D,MAAM,CAAC,IAAI,EAAE,CAAC,8BAA8B,EAAE,EAAE,GAAG,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,CAAC;QACnE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,GAAG,EAAE,KAAK,EAAE,iBAAiB,GAAG,EAAE,EAAE,CAAC;IAC/D,CAAC;IAED,IAAI,CAAC,IAAI,CAAC,EAAE,EAAE,CAAC;QACb,MAAM,IAAI,CAAC,IAAI,EAAE,MAAM,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;QAC1C,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,GAAG,EAAE,KAAK,EAAE,QAAQ,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,UAAU,EAAE,EAAE,CAAC;IACjF,CAAC;IAED,MAAM,WAAW,GAAG,mBAAmB,CACrC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,IAAI,EAAE,CACvC,CAAC;IAEF,IAAI,QAAgB,CAAC;IACrB,IAAI,CAAC;QACH,QAAQ,GAAG,MAAM,cAAc,CAAC,IAAI,CAAC,CAAC;IACxC,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,MAAM,GAAG,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QAC7D,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,GAAG,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC;IAC5C,CAAC;YAAS,CAAC;QACT,MAAM,IAAI,CAAC,IAAI,EAAE,MAAM,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;IAC5C,CAAC;IAED,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,EAAE,CAAC;QACrB,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,GAAG,EAAE,KAAK,EAAE,YAAY,EAAE,CAAC;IACrD,CAAC;IAED,6CAA6C;IAC7C,MAAM,MAAM,GAAG,WAAW,CACxB,QAAQ,EACR,WAAW,EACX,GAAG,EACH,MAAM,EACN,SAAS,EACT,QAAQ,CACT,CAAC;IAEF,wEAAwE;IACxE,8DAA8D;IAC9D,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC;QAC/B,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,GAAG,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YAC7D,MAAM,CAAC,KAAK,CAAC,mCAAmC,EAAE,GAAY,EAAE;gBAC9D,GAAG;gBACH,OAAO,EAAE,KAAK,CAAC,EAAE;aAClB,CAAC,CAAC;YACH,OAAO;gBACL,IAAI,EAAE,OAAO;gBACb,GAAG;gBACH,KAAK,EAAE,4BAA4B,KAAK,CAAC,EAAE,KAAK,GAAG,EAAE;aACtD,CAAC;QACJ,CAAC;IACH,CAAC;IAED,IAAI,CAAC,WAAW,EAAE,CAAC,QAAQ,EAAE,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;IAEzC,OAAO;QACL,IAAI,EAAE,SAAS;QACf,GAAG;QACH,MAAM,EAAE,MAAM,IAAI,GAAG;QACrB,WAAW;QACX,MAAM;QACN,KAAK,EAAE,MAAM,CAAC,UAAU,CAAC,QAAQ,EAAE,OAAO,CAAC;KAC5C,CAAC;AACJ,CAAC;AAED;;;;;GAKG;AACH,SAAS,WAAW,CAClB,IAAY,EACZ,WAA6B,EAC7B,GAAW,EACX,MAA0B,EAC1B,SAAiB,EACjB,QAAgB;IAEhB,MAAM,QAAQ,GAAG;QACf,SAAS;QACT,MAAM,EAAE,MAAM,IAAI,GAAG;QACrB,GAAG;QACH,QAAQ;QACR,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;KACpC,CAAC;IACF,MAAM,OAAO,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IAE5E,IAAI,WAAW,KAAK,MAAM,EAAE,CAAC;QAC3B,IAAI,MAAM,GAAY,IAAI,CAAC;QAC3B,IAAI,CAAC;YACH,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAC5B,CAAC;QAAC,MAAM,CAAC;YACP,qDAAqD;YACrD,MAAM,GAAG,IAAI,CAAC;QAChB,CAAC;QACD,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,OAAO,MAAM,KAAK,SAAS,EAAE,CAAC;YAC5F,OAAO;gBACL;oBACE,EAAE,EAAE,GAAG,SAAS,QAAQ,OAAO,IAAI;oBACnC,OAAO,EAAE,aAAa,MAAM,CAAC,MAAM,CAAC,kBAAkB,GAAG,GAAG;oBAC5D,QAAQ,EAAE,EAAE,GAAG,QAAQ,EAAE,IAAI,EAAE,WAAW,EAAE,KAAK,EAAE,GAAG,EAAE;iBACzD;aACF,CAAC;QACJ,CAAC;QACD,MAAM,EAAE,GAAG,mBAAmB,CAAC,MAAM,CAAC,CAAC;QACvC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACpB,OAAO;gBACL;oBACE,EAAE,EAAE,GAAG,SAAS,QAAQ,OAAO,IAAI;oBACnC,OAAO,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC;oBAC5B,QAAQ,EAAE,EAAE,GAAG,QAAQ,EAAE,IAAI,EAAE,WAAW,EAAE;iBAC7C;aACF,CAAC;QACJ,CAAC;QACD,OAAO,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;YACvB,EAAE,EAAE,GAAG,SAAS,QAAQ,OAAO,IAAI,CAAC,EAAE;YACtC,OAAO,EAAE,GAAG,CAAC,CAAC,OAAO,gBAAgB,GAAG,GAAG;YAC3C,QAAQ,EAAE,EAAE,GAAG,QAAQ,EAAE,IAAI,EAAE,UAAU,EAAE,KAAK,EAAE,CAAC,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE;SACzE,CAAC,CAAC,CAAC;IACN,CAAC;IAED,IAAI,WAAW,KAAK,MAAM,EAAE,CAAC;QAC3B,MAAM,EAAE,GAAG,cAAc,CAAC,IAAI,CAAC,CAAC;QAChC,IAAI,CAAC,EAAE;YAAE,OAAO,EAAE,CAAC;QACnB,yEAAyE;QACzE,qEAAqE;QACrE,OAAO,aAAa,CAAC,EAAE,EAAE,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;YAChD,EAAE,EAAE,GAAG,SAAS,QAAQ,OAAO,IAAI,CAAC,EAAE;YACtC,OAAO,EAAE,YAAY,GAAG,QAAQ,KAAK,EAAE;YACvC,QAAQ,EAAE,EAAE,GAAG,QAAQ,EAAE,IAAI,EAAE,UAAU,EAAE,UAAU,EAAE,CAAC,EAAE;SAC3D,CAAC,CAAC,CAAC;IACN,CAAC;IAED,gCAAgC;IAChC,OAAO;QACL;YACE,EAAE,EAAE,GAAG,SAAS,QAAQ,OAAO,IAAI;YACnC,OAAO,EAAE,YAAY,GAAG,QAAQ,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,EAAE;YACrD,QAAQ,EAAE,EAAE,GAAG,QAAQ,EAAE,IAAI,EAAE,UAAU,EAAE;SAC5C;KACF,CAAC;AACJ,CAAC;AAED,sEAAsE;AACtE,SAAS,aAAa,CAAC,EAAU,EAAE,IAAY;IAC7C,IAAI,EAAE,CAAC,MAAM,IAAI,IAAI;QAAE,OAAO,CAAC,EAAE,CAAC,CAAC;IACnC,MAAM,GAAG,GAAa,EAAE,CAAC;IACzB,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,OAAO,KAAK,GAAG,EAAE,CAAC,MAAM,EAAE,CAAC;QACzB,IAAI,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,GAAG,IAAI,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC;QAC5C,IAAI,GAAG,GAAG,EAAE,CAAC,MAAM,EAAE,CAAC;YACpB,wEAAwE;YACxE,MAAM,WAAW,GAAG,EAAE,CAAC,WAAW,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;YAChD,IAAI,WAAW,GAAG,KAAK,GAAG,IAAI,GAAG,GAAG;gBAAE,GAAG,GAAG,WAAW,GAAG,CAAC,CAAC;QAC9D,CAAC;QACD,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,CAAC;QAC/B,KAAK,GAAG,GAAG,CAAC;IACd,CAAC;IACD,OAAO,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;AAChD,CAAC"}

package/dist/services/web/html-to-md.d.ts

@@ -0,0 +1,38 @@
+/**
+ * HTML → Markdown conversion + JSON key-path chunking for `fetch_and_index`.
+ *
+ * - `htmlToMarkdown(html)` strips scripts/styles/nav/etc. and converts the rest
+ *   to GitHub-Flavored Markdown (tables, strikethrough, fenced code) via
+ *   `turndown` + `turndown-plugin-gfm`. Turndown 7.x bundles `@mixmark-io/domino`
+ *   as a DOM implementation, so it works in Bun without a global DOM.
+ *
+ * - `jsonToKeyPathChunks(json, label?)` walks a parsed JSON value and emits one
+ *   markdown chunk per leaf path (e.g. `user.address.city → "Lyon"`). Arrays of
+ *   primitives become a bulleted list under their path; arrays of objects fan
+ *   out one entry per element. This gives each value a searchable key-path
+ *   anchor instead of dumping pretty-printed JSON into one undifferentiated blob.
+ */
+/**
+ * Convert an HTML string to GitHub-Flavored Markdown. Scripts, styles, nav,
+ * header, footer, noscript, and iframes are stripped before conversion.
+ * Returns the empty string for falsy / whitespace-only input so the caller can
+ * treat it as "nothing to index".
+ */
+export declare function htmlToMarkdown(html: string): string;
+export interface JsonChunk {
+    /** The leaf path, e.g. `items[0].name`. */
+    path: string;
+    /** Rendered as markdown: `**items[0].name** = \`Lyon\``. */
+    content: string;
+}
+/**
+ * Walk a parsed JSON value and emit one chunk per leaf. Object properties
+ * descend by key; arrays of objects fan out one entry per element (indexed);
+ * arrays of primitives collapse to a single bulleted-list chunk under their
+ * path. `null`/`undefined` leaves are skipped (no value to search).
+ *
+ * Each chunk's `content` is a small markdown snippet with the path bolded as a
+ * key anchor so both vector and keyword search can match on the path name.
+ */
+export declare function jsonToKeyPathChunks(value: unknown, label?: string): JsonChunk[];
+//# sourceMappingURL=html-to-md.d.ts.map

package/dist/services/web/html-to-md.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"html-to-md.d.ts","sourceRoot":"","sources":["../../../src/services/web/html-to-md.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAoCH;;;;;GAKG;AACH,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAKnD;AAED,MAAM,WAAW,SAAS;IACxB,2CAA2C;IAC3C,IAAI,EAAE,MAAM,CAAC;IACb,4DAA4D;IAC5D,OAAO,EAAE,MAAM,CAAC;CACjB;AAED;;;;;;;;GAQG;AACH,wBAAgB,mBAAmB,CAAC,KAAK,EAAE,OAAO,EAAE,KAAK,SAAM,GAAG,SAAS,EAAE,CAI5E"}

package/dist/services/web/html-to-md.js

@@ -0,0 +1,98 @@
+/**
+ * HTML → Markdown conversion + JSON key-path chunking for `fetch_and_index`.
+ *
+ * - `htmlToMarkdown(html)` strips scripts/styles/nav/etc. and converts the rest
+ *   to GitHub-Flavored Markdown (tables, strikethrough, fenced code) via
+ *   `turndown` + `turndown-plugin-gfm`. Turndown 7.x bundles `@mixmark-io/domino`
+ *   as a DOM implementation, so it works in Bun without a global DOM.
+ *
+ * - `jsonToKeyPathChunks(json, label?)` walks a parsed JSON value and emits one
+ *   markdown chunk per leaf path (e.g. `user.address.city → "Lyon"`). Arrays of
+ *   primitives become a bulleted list under their path; arrays of objects fan
+ *   out one entry per element. This gives each value a searchable key-path
+ *   anchor instead of dumping pretty-printed JSON into one undifferentiated blob.
+ */
+import TurndownService from "turndown";
+// eslint-disable-next-line @typescript-eslint/no-require-imports
+const gfm = require("turndown-plugin-gfm").gfm;
+const STRIP_SELECTORS = ["script", "style", "nav", "header", "footer", "noscript", "iframe"];
+/** Lazy singleton TurndownService — built once, reused across calls. */
+let tdCache = null;
+function getTurndown() {
+    if (tdCache)
+        return tdCache;
+    const td = new TurndownService({
+        headingStyle: "atx",
+        codeBlockStyle: "fenced",
+        bulletListMarker: "-",
+        emDelimiter: "_",
+    });
+    td.use(gfm);
+    // `remove()` drops the matched elements entirely (contents included) before
+    // conversion — so inline scripts/styles never reach the markdown output.
+    td.remove(STRIP_SELECTORS);
+    tdCache = td;
+    return td;
+}
+/**
+ * Convert an HTML string to GitHub-Flavored Markdown. Scripts, styles, nav,
+ * header, footer, noscript, and iframes are stripped before conversion.
+ * Returns the empty string for falsy / whitespace-only input so the caller can
+ * treat it as "nothing to index".
+ */
+export function htmlToMarkdown(html) {
+    if (!html || !html.trim())
+        return "";
+    const md = getTurndown().turndown(html);
+    // Collapse runs of blank lines turndown sometimes leaves (3+ → 2) and trim.
+    return md.replace(/\n{3,}/g, "\n\n").trim();
+}
+/**
+ * Walk a parsed JSON value and emit one chunk per leaf. Object properties
+ * descend by key; arrays of objects fan out one entry per element (indexed);
+ * arrays of primitives collapse to a single bulleted-list chunk under their
+ * path. `null`/`undefined` leaves are skipped (no value to search).
+ *
+ * Each chunk's `content` is a small markdown snippet with the path bolded as a
+ * key anchor so both vector and keyword search can match on the path name.
+ */
+export function jsonToKeyPathChunks(value, label = "$") {
+    const out = [];
+    walk(value, label, out);
+    return out;
+}
+function walk(val, path, out) {
+    if (val === null || val === undefined)
+        return;
+    if (Array.isArray(val)) {
+        if (val.length === 0) {
+            out.push({ path, content: `**${path}** = _[]_` });
+            return;
+        }
+        // Array of objects → fan out one entry per element (keeps each object's
+        // fields independently searchable under a stable indexed path).
+        if (val.every((v) => v !== null && typeof v === "object")) {
+            val.forEach((v, i) => walk(v, `${path}[${i}]`, out));
+            return;
+        }
+        // Array of primitives → one bulleted list chunk under the array path.
+        const items = val.map((v) => `- \`${String(v)}\``).join("\n");
+        out.push({ path, content: `**${path}**\n\n${items}` });
+        return;
+    }
+    if (typeof val === "object") {
+        const entries = Object.entries(val);
+        if (entries.length === 0) {
+            out.push({ path, content: `**${path}** = _{}_` });
+            return;
+        }
+        for (const [k, v] of entries) {
+            const safeKey = /^[A-Za-z_$][\w$]*$/.test(k) ? k : JSON.stringify(k);
+            walk(v, `${path}.${safeKey}`, out);
+        }
+        return;
+    }
+    // Leaf primitive.
+    out.push({ path, content: `**${path}** = \`${String(val)}\`` });
+}
+//# sourceMappingURL=html-to-md.js.map

package/dist/services/web/html-to-md.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"html-to-md.js","sourceRoot":"","sources":["../../../src/services/web/html-to-md.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAEH,OAAO,eAAe,MAAM,UAAU,CAAC;AAWvC,iEAAiE;AACjE,MAAM,GAAG,GAAG,OAAO,CAAC,qBAAqB,CAAC,CAAC,GAAgB,CAAC;AAE5D,MAAM,eAAe,GAAG,CAAC,QAAQ,EAAE,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,QAAQ,EAAE,UAAU,EAAE,QAAQ,CAAC,CAAC;AAE7F,wEAAwE;AACxE,IAAI,OAAO,GAA2B,IAAI,CAAC;AAC3C,SAAS,WAAW;IAClB,IAAI,OAAO;QAAE,OAAO,OAAO,CAAC;IAC5B,MAAM,EAAE,GAAG,IAAI,eAAe,CAAC;QAC7B,YAAY,EAAE,KAAK;QACnB,cAAc,EAAE,QAAQ;QACxB,gBAAgB,EAAE,GAAG;QACrB,WAAW,EAAE,GAAG;KACjB,CAAC,CAAC;IACH,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IACZ,4EAA4E;IAC5E,yEAAyE;IACzE,EAAE,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC;IAC3B,OAAO,GAAG,EAAE,CAAC;IACb,OAAO,EAAE,CAAC;AACZ,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,cAAc,CAAC,IAAY;IACzC,IAAI,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE;QAAE,OAAO,EAAE,CAAC;IACrC,MAAM,EAAE,GAAG,WAAW,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;IACxC,4EAA4E;IAC5E,OAAO,EAAE,CAAC,OAAO,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC,IAAI,EAAE,CAAC;AAC9C,CAAC;AASD;;;;;;;;GAQG;AACH,MAAM,UAAU,mBAAmB,CAAC,KAAc,EAAE,KAAK,GAAG,GAAG;IAC7D,MAAM,GAAG,GAAgB,EAAE,CAAC;IAC5B,IAAI,CAAC,KAAK,EAAE,KAAK,EAAE,GAAG,CAAC,CAAC;IACxB,OAAO,GAAG,CAAC;AACb,CAAC;AAED,SAAS,IAAI,CAAC,GAAY,EAAE,IAAY,EAAE,GAAgB;IACxD,IAAI,GAAG,KAAK,IAAI,IAAI,GAAG,KAAK,SAAS;QAAE,OAAO;IAE9C,IAAI,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC;QACvB,IAAI,GAAG,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACrB,GAAG,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,IAAI,WAAW,EAAE,CAAC,CAAC;YAClD,OAAO;QACT,CAAC;QACD,wEAAwE;QACxE,gEAAgE;QAChE,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,IAAI,IAAI,OAAO,CAAC,KAAK,QAAQ,CAAC,EAAE,CAAC;YAC1D,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,CAAC,EAAE,GAAG,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC;YACrD,OAAO;QACT,CAAC;QACD,sEAAsE;QACtE,MAAM,KAAK,GAAG,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC9D,GAAG,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,IAAI,SAAS,KAAK,EAAE,EAAE,CAAC,CAAC;QACvD,OAAO;IACT,CAAC;IAED,IAAI,OAAO,GAAG,KAAK,QAAQ,EAAE,CAAC;QAC5B,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC,GAA8B,CAAC,CAAC;QAC/D,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACzB,GAAG,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,IAAI,WAAW,EAAE,CAAC,CAAC;YAClD,OAAO;QACT,CAAC;QACD,KAAK,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,OAAO,EAAE,CAAC;YAC7B,MAAM,OAAO,GAAG,oBAAoB,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC;YACrE,IAAI,CAAC,CAAC,EAAE,GAAG,IAAI,IAAI,OAAO,EAAE,EAAE,GAAG,CAAC,CAAC;QACrC,CAAC;QACD,OAAO;IACT,CAAC;IAED,kBAAkB;IAClB,GAAG,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,IAAI,UAAU,MAAM,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC;AAClE,CAAC"}

package/dist/services/web/index.d.ts

@@ -0,0 +1,15 @@
+/**
+ * Web ingestion service — SSRF-guarded fetch + HTML→md + index.
+ *
+ * Public surface:
+ *   - classifyIp / assertUrlSafe / fetchWithSsrfGuard / SsrfBlockedError (ssrf)
+ *   - htmlToMarkdown / jsonToKeyPathChunks (html-to-md)
+ *   - fetchAndConvertOne / composeFetchCacheKey / MAX_FETCH_BYTES /
+ *     DEFAULT_FETCH_TTL_MS + types (fetcher)
+ *   - WebController (orchestration)
+ */
+export { classifyIp, assertUrlSafe, fetchWithSsrfGuard, SsrfBlockedError, setDnsResolver, MAX_REDIRECTS, type IpClass, type FetchGuardOptions, type DnsResolver, type UrlSafetyResult, } from "./ssrf.js";
+export { htmlToMarkdown, jsonToKeyPathChunks, type JsonChunk, } from "./html-to-md.js";
+export { fetchAndConvertOne, composeFetchCacheKey, MAX_FETCH_BYTES, DEFAULT_FETCH_TTL_MS, type FetchContentType, type FetchOneResult, type FetchOneOptions, type IndexedChunk, type WebIndexDeps, } from "./fetcher.js";
+export { WebController, type FetchRequest, type FetchAndIndexParams, type WebControllerDeps, } from "./web-controller.js";
+//# sourceMappingURL=index.d.ts.map

package/dist/services/web/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/services/web/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAEH,OAAO,EACL,UAAU,EACV,aAAa,EACb,kBAAkB,EAClB,gBAAgB,EAChB,cAAc,EACd,aAAa,EACb,KAAK,OAAO,EACZ,KAAK,iBAAiB,EACtB,KAAK,WAAW,EAChB,KAAK,eAAe,GACrB,MAAM,WAAW,CAAC;AAEnB,OAAO,EACL,cAAc,EACd,mBAAmB,EACnB,KAAK,SAAS,GACf,MAAM,iBAAiB,CAAC;AAEzB,OAAO,EACL,kBAAkB,EAClB,oBAAoB,EACpB,eAAe,EACf,oBAAoB,EACpB,KAAK,gBAAgB,EACrB,KAAK,cAAc,EACnB,KAAK,eAAe,EACpB,KAAK,YAAY,EACjB,KAAK,YAAY,GAClB,MAAM,cAAc,CAAC;AAEtB,OAAO,EACL,aAAa,EACb,KAAK,YAAY,EACjB,KAAK,mBAAmB,EACxB,KAAK,iBAAiB,GACvB,MAAM,qBAAqB,CAAC"}