@multiplayer-app/agent 0.1.10 → 0.1.11

package/dist/research/extract-texts.d.ts

@@ -0,0 +1,41 @@
+import { type LanguageModel } from 'ai';
+export declare const EXTRACTION_CHUNK_CHARS = 60000;
+export interface ExtractionRequest {
+    paths?: string[];
+    manifest?: string;
+    question: string;
+    fields: string[];
+    outputPath: string;
+    windows?: {
+        patterns: string[];
+        radius?: number;
+        maxWindows?: number;
+    };
+    many?: boolean;
+    allowed?: Record<string, string[]>;
+    concurrency?: number;
+    dryRun?: boolean;
+}
+export interface ExtractionDeps {
+    workspaceDir: string;
+    model: LanguageModel;
+    modelId: string;
+    signal?: AbortSignal;
+    recordUsage: (usage: {
+        inputTokens: number;
+        outputTokens: number;
+        totalTokens: number;
+        cacheReadTokens?: number;
+        cacheWriteTokens?: number;
+    }) => void;
+    onProgress?: (done: number, total: number) => void;
+}
+/** Passages around matches, merged when they overlap, each labelled with its character offset. */
+export declare function windowsOf(text: string, patterns: string[], radius?: number, maxWindows?: number): {
+    passages: string[];
+    matches: number;
+};
+/** Every requested field present (null when missing); values outside the codebook are listed in `invalid`. */
+export declare function normalizeRow(row: Record<string, unknown>, fields: string[], allowed?: Record<string, string[]>): Record<string, unknown>;
+export declare function extractFromTexts(request: ExtractionRequest, deps: ExtractionDeps): Promise<Record<string, unknown>>;
+//# sourceMappingURL=extract-texts.d.ts.map

package/dist/research/extract-texts.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"extract-texts.d.ts","sourceRoot":"","sources":["../../src/research/extract-texts.ts"],"names":[],"mappings":"AASA,OAAO,EAAgB,KAAK,aAAa,EAAE,MAAM,IAAI,CAAA;AAGrD,eAAO,MAAM,sBAAsB,QAAS,CAAA;AAa5C,MAAM,WAAW,iBAAiB;IAChC,KAAK,CAAC,EAAE,MAAM,EAAE,CAAA;IAChB,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB,QAAQ,EAAE,MAAM,CAAA;IAChB,MAAM,EAAE,MAAM,EAAE,CAAA;IAChB,UAAU,EAAE,MAAM,CAAA;IAClB,OAAO,CAAC,EAAE;QAAE,QAAQ,EAAE,MAAM,EAAE,CAAC;QAAC,MAAM,CAAC,EAAE,MAAM,CAAC;QAAC,UAAU,CAAC,EAAE,MAAM,CAAA;KAAE,CAAA;IACtE,IAAI,CAAC,EAAE,OAAO,CAAA;IACd,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,CAAA;IAClC,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,MAAM,CAAC,EAAE,OAAO,CAAA;CACjB;AAED,MAAM,WAAW,cAAc;IAC7B,YAAY,EAAE,MAAM,CAAA;IACpB,KAAK,EAAE,aAAa,CAAA;IACpB,OAAO,EAAE,MAAM,CAAA;IACf,MAAM,CAAC,EAAE,WAAW,CAAA;IACpB,WAAW,EAAE,CAAC,KAAK,EAAE;QAAE,WAAW,EAAE,MAAM,CAAC;QAAC,YAAY,EAAE,MAAM,CAAC;QAAC,WAAW,EAAE,MAAM,CAAC;QAAC,eAAe,CAAC,EAAE,MAAM,CAAC;QAAC,gBAAgB,CAAC,EAAE,MAAM,CAAA;KAAE,KAAK,IAAI,CAAA;IACrJ,UAAU,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,KAAK,IAAI,CAAA;CACnD;AAsBD,kGAAkG;AAClG,wBAAgB,SAAS,CAAC,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE,MAAM,SAAwB,EAAE,UAAU,SAAsB,GAAG;IAAE,QAAQ,EAAE,MAAM,EAAE,CAAC;IAAC,OAAO,EAAE,MAAM,CAAA;CAAE,CAerK;AAQD,8GAA8G;AAC9G,wBAAgB,YAAY,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,EAAE,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAQxI;AAmBD,wBAAsB,gBAAgB,CAAC,OAAO,EAAE,iBAAiB,EAAE,IAAI,EAAE,cAAc,GAAG,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CA0HzH"}

package/dist/research/extract-texts.js

@@ -0,0 +1,239 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.EXTRACTION_CHUNK_CHARS = void 0;
+exports.windowsOf = windowsOf;
+exports.normalizeRow = normalizeRow;
+exports.extractFromTexts = extractFromTexts;
+// Bulk extraction on the cheaper model: the same question over many extracted-text files, one
+// JSON row (or several) per document appended to a JSONL file. The main model never sees the
+// documents, only the counts; it aggregates the rows with run_command.
+//
+// What reaches the extraction model: with `windows`, only the passages around pattern matches
+// (a 300-page rules manual becomes a few kilobytes, and a document with no match costs nothing);
+// without, the whole document in chunks, never a silent cut at the first N characters.
+const fs_1 = require("fs");
+const path_1 = __importDefault(require("path"));
+const ai_1 = require("ai");
+const filingramp_tools_1 = require("./filingramp-tools");
+exports.EXTRACTION_CHUNK_CHARS = 60000;
+const DEFAULT_WINDOW_RADIUS = 1500;
+const DEFAULT_MAX_WINDOWS = 6;
+const DEFAULT_CONCURRENCY = 4;
+/** Prompt overhead per call (system + question + fields), and a typical JSON answer, in tokens. */
+const CALL_OVERHEAD_TOKENS = 350;
+const ANSWER_TOKENS = 250;
+const SYSTEM_ONE = 'You extract facts from one insurance filing document (or passages of it). Answer with a single JSON object only — no prose. Use null for a field the text does not state. Always include "quote": the exact supporting text (max 300 characters) or null, and "found": true|false.';
+const SYSTEM_MANY = 'You extract facts from one insurance filing document (or passages of it). The document may state the fact several times (for example one notice period per notice type). Answer with JSON only: {"rows": [ ... ]}, one object per distinct statement, each with the requested fields, "quote" (exact supporting text, max 300 characters) and "found": true. Use null for a field a statement does not give. No statement → {"rows": []}.';
+function inside(workspaceDir, relative) {
+    const target = path_1.default.resolve(workspaceDir, relative);
+    if (!target.startsWith(path_1.default.resolve(workspaceDir) + path_1.default.sep))
+        throw new Error(`path escapes the workspace: ${relative}`);
+    return target;
+}
+/** Text files a manifest row points at: textPath (fetch_attachments, saveHits) or a .txt path. */
+async function sourcesOf(workspaceDir, request) {
+    var _a;
+    const sources = [...((_a = request.paths) !== null && _a !== void 0 ? _a : [])];
+    let withoutText = 0;
+    if (request.manifest) {
+        for (const row of await (0, filingramp_tools_1.readManifest)(workspaceDir, request.manifest)) {
+            const candidate = typeof row.textPath === 'string' ? row.textPath : typeof row.path === 'string' && row.path.endsWith('.txt') ? row.path : undefined;
+            if (candidate)
+                sources.push(candidate);
+            else
+                withoutText += 1;
+        }
+    }
+    return { sources: [...new Set(sources)], withoutText };
+}
+/** Passages around matches, merged when they overlap, each labelled with its character offset. */
+function windowsOf(text, patterns, radius = DEFAULT_WINDOW_RADIUS, maxWindows = DEFAULT_MAX_WINDOWS) {
+    const regex = new RegExp(patterns.map((pattern) => `(?:${pattern})`).join('|'), 'gi');
+    const spans = [];
+    let matches = 0;
+    for (const match of text.matchAll(regex)) {
+        if (match[0].length === 0)
+            break;
+        matches += 1;
+        const start = Math.max(0, match.index - radius);
+        const end = Math.min(text.length, match.index + match[0].length + radius);
+        const last = spans[spans.length - 1];
+        if (last && start <= last[1])
+            last[1] = Math.max(last[1], end);
+        else
+            spans.push([start, end]);
+    }
+    const passages = spans.slice(0, maxWindows).map(([start, end]) => `[chars ${start}–${end}]\n${text.slice(start, end)}`);
+    return { passages, matches };
+}
+function chunksOf(text) {
+    const chunks = [];
+    for (let start = 0; start < text.length; start += exports.EXTRACTION_CHUNK_CHARS)
+        chunks.push(text.slice(start, start + exports.EXTRACTION_CHUNK_CHARS));
+    return chunks.length ? chunks : [''];
+}
+/** Every requested field present (null when missing); values outside the codebook are listed in `invalid`. */
+function normalizeRow(row, fields, allowed) {
+    const out = { ...row };
+    for (const field of fields)
+        if (!(field in out))
+            out[field] = null;
+    const invalid = Object.entries(allowed !== null && allowed !== void 0 ? allowed : {})
+        .filter(([field, values]) => out[field] !== null && out[field] !== undefined && !values.includes(String(out[field])))
+        .map(([field]) => field);
+    if (invalid.length)
+        out.invalid = invalid;
+    return out;
+}
+function parseAnswer(text, many) {
+    const match = text.match(/\{[\s\S]*\}/);
+    if (!match)
+        throw new Error('the extraction model answered without JSON');
+    const parsed = JSON.parse(match[0]);
+    if (!many)
+        return [parsed];
+    return Array.isArray(parsed.rows) ? parsed.rows : [];
+}
+async function doneSources(output) {
+    try {
+        const raw = await fs_1.promises.readFile(output, 'utf8');
+        return new Set(raw.split('\n').filter(Boolean).map((line) => String(JSON.parse(line).source)));
+    }
+    catch (_a) {
+        return new Set();
+    }
+}
+async function extractFromTexts(request, deps) {
+    var _a, _b, _c, _d;
+    if (!((_a = request.paths) === null || _a === void 0 ? void 0 : _a.length) && !request.manifest)
+        throw new Error('Pass paths or a manifest.');
+    const { sources, withoutText } = await sourcesOf(deps.workspaceDir, request);
+    const output = inside(deps.workspaceDir, request.outputPath);
+    const already = await doneSources(output);
+    const pending = sources.filter((source) => !already.has(source));
+    const many = (_b = request.many) !== null && _b !== void 0 ? _b : false;
+    const prepare = async (source) => {
+        var _a;
+        const text = await fs_1.promises.readFile(inside(deps.workspaceDir, source), 'utf8');
+        if (!((_a = request.windows) === null || _a === void 0 ? void 0 : _a.patterns.length))
+            return { parts: chunksOf(text), matches: undefined };
+        const { passages, matches } = windowsOf(text, request.windows.patterns, request.windows.radius, request.windows.maxWindows);
+        return { parts: passages.length ? [passages.join('\n…\n')] : [], matches };
+    };
+    if (request.dryRun) {
+        let calls = 0;
+        let chars = 0;
+        let noMatch = 0;
+        for (const source of pending) {
+            const { parts } = await prepare(source).catch(() => ({ parts: [] }));
+            if (!parts.length)
+                noMatch += 1;
+            calls += parts.length;
+            chars += parts.reduce((sum, part) => sum + part.length, 0);
+        }
+        return {
+            dryRun: true,
+            documents: pending.length,
+            alreadyDone: sources.length - pending.length,
+            manifestRowsWithoutText: withoutText,
+            noMatch,
+            calls,
+            inputChars: chars,
+            estimatedTokens: Math.round(chars / 4) + calls * (CALL_OVERHEAD_TOKENS + ANSWER_TOKENS),
+            model: deps.modelId,
+        };
+    }
+    await fs_1.promises.mkdir(path_1.default.dirname(output), { recursive: true });
+    const failures = [];
+    let calls = 0;
+    let rows = 0;
+    let found = 0;
+    let noMatch = 0;
+    let inputChars = 0;
+    let done = 0;
+    const append = async (row) => {
+        await fs_1.promises.appendFile(output, `${JSON.stringify(row)}\n`);
+        rows += 1;
+        if (row.found === true)
+            found += 1;
+    };
+    const queue = [...pending];
+    const worker = async () => {
+        var _a, _b, _c, _d, _e, _f, _g, _h;
+        while (queue.length) {
+            if ((_a = deps.signal) === null || _a === void 0 ? void 0 : _a.aborted)
+                return;
+            const source = queue.shift();
+            try {
+                const { parts, matches } = await prepare(source);
+                if (!parts.length) {
+                    // No pattern match: nothing to send, the row still records that the document was checked.
+                    noMatch += 1;
+                    await append(normalizeRow({ source, found: false, quote: null, reason: 'no pattern match' }, request.fields));
+                    continue;
+                }
+                const docRows = [];
+                for (const [index, part] of parts.entries()) {
+                    calls += 1;
+                    inputChars += part.length;
+                    const result = await (0, ai_1.generateText)({
+                        model: deps.model,
+                        abortSignal: deps.signal,
+                        system: many ? SYSTEM_MANY : SYSTEM_ONE,
+                        prompt: `Question: ${request.question}\nFields: ${request.fields.join(', ')}${request.allowed ? `\nAllowed values: ${JSON.stringify(request.allowed)}` : ''}\n\nDocument (${source}${parts.length > 1 ? `, part ${index + 1} of ${parts.length}` : ''}${matches !== undefined ? `, ${matches} matching passages` : ''}):\n${part}`,
+                    });
+                    const usage = result.usage;
+                    if (usage) {
+                        const inputTokens = (_b = usage.inputTokens) !== null && _b !== void 0 ? _b : 0;
+                        const outputTokens = (_c = usage.outputTokens) !== null && _c !== void 0 ? _c : 0;
+                        deps.recordUsage({
+                            inputTokens,
+                            outputTokens,
+                            totalTokens: (_d = usage.totalTokens) !== null && _d !== void 0 ? _d : inputTokens + outputTokens,
+                            cacheReadTokens: (_e = usage.inputTokenDetails) === null || _e === void 0 ? void 0 : _e.cacheReadTokens,
+                            cacheWriteTokens: (_f = usage.inputTokenDetails) === null || _f === void 0 ? void 0 : _f.cacheWriteTokens,
+                        });
+                    }
+                    for (const row of parseAnswer(result.text, many))
+                        docRows.push({ source, ...(parts.length > 1 ? { part: index + 1 } : {}), ...row });
+                }
+                // One row per document: the first part that found it, else the model's own answer for the
+                // first part. Never an empty stand-in: a summary question fills its fields even when the
+                // model sets found=false, and dropping them loses the extraction.
+                const kept = many ? docRows : [(_g = docRows.find((row) => row.found === true)) !== null && _g !== void 0 ? _g : docRows[0]].filter(Boolean);
+                const final = kept.length ? kept : [{ source, found: false, quote: null }];
+                for (const row of final)
+                    await append(normalizeRow(row, request.fields, request.allowed));
+            }
+            catch (error) {
+                failures.push({ path: source, reason: error.message });
+            }
+            finally {
+                done += 1;
+                if (done % 5 === 0 || done === pending.length)
+                    (_h = deps.onProgress) === null || _h === void 0 ? void 0 : _h.call(deps, done, pending.length);
+            }
+        }
+    };
+    (_c = deps.onProgress) === null || _c === void 0 ? void 0 : _c.call(deps, 0, pending.length);
+    await Promise.all(Array.from({ length: Math.min((_d = request.concurrency) !== null && _d !== void 0 ? _d : DEFAULT_CONCURRENCY, Math.max(1, pending.length)) }, worker));
+    return {
+        outputPath: request.outputPath,
+        documents: pending.length,
+        alreadyDone: sources.length - pending.length,
+        manifestRowsWithoutText: withoutText,
+        noMatch,
+        calls,
+        rows,
+        found,
+        failed: failures.length,
+        failures: failures.slice(0, 5),
+        inputChars,
+        model: deps.modelId,
+        next: 'Aggregate the JSONL with run_command (python/jq); read only the rows you need to check.',
+    };
+}
+//# sourceMappingURL=extract-texts.js.map

package/dist/research/extract-texts.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"extract-texts.js","sourceRoot":"","sources":["../../src/research/extract-texts.ts"],"names":[],"mappings":";;;;;;AAoEA,8BAeC;AASD,oCAQC;AAmBD,4CA0HC;AAjPD,8FAA8F;AAC9F,6FAA6F;AAC7F,uEAAuE;AACvE,EAAE;AACF,8FAA8F;AAC9F,iGAAiG;AACjG,uFAAuF;AACvF,2BAAmC;AACnC,gDAAuB;AACvB,2BAAqD;AACrD,yDAAiD;AAEpC,QAAA,sBAAsB,GAAG,KAAM,CAAA;AAC5C,MAAM,qBAAqB,GAAG,IAAK,CAAA;AACnC,MAAM,mBAAmB,GAAG,CAAC,CAAA;AAC7B,MAAM,mBAAmB,GAAG,CAAC,CAAA;AAC7B,mGAAmG;AACnG,MAAM,oBAAoB,GAAG,GAAG,CAAA;AAChC,MAAM,aAAa,GAAG,GAAG,CAAA;AAEzB,MAAM,UAAU,GACd,oRAAoR,CAAA;AACtR,MAAM,WAAW,GACf,2aAA2a,CAAA;AAwB7a,SAAS,MAAM,CAAC,YAAoB,EAAE,QAAgB;IACpD,MAAM,MAAM,GAAG,cAAI,CAAC,OAAO,CAAC,YAAY,EAAE,QAAQ,CAAC,CAAA;IACnD,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,cAAI,CAAC,OAAO,CAAC,YAAY,CAAC,GAAG,cAAI,CAAC,GAAG,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,+BAA+B,QAAQ,EAAE,CAAC,CAAA;IACzH,OAAO,MAAM,CAAA;AACf,CAAC;AAED,kGAAkG;AAClG,KAAK,UAAU,SAAS,CAAC,YAAoB,EAAE,OAA0B;;IACvE,MAAM,OAAO,GAAG,CAAC,GAAG,CAAC,MAAA,OAAO,CAAC,KAAK,mCAAI,EAAE,CAAC,CAAC,CAAA;IAC1C,IAAI,WAAW,GAAG,CAAC,CAAA;IACnB,IAAI,OAAO,CAAC,QAAQ,EAAE,CAAC;QACrB,KAAK,MAAM,GAAG,IAAI,MAAM,IAAA,+BAAY,EAAC,YAAY,EAAE,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAC;YACrE,MAAM,SAAS,GAAG,OAAO,GAAG,CAAC,QAAQ,KAAK,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,OAAO,GAAG,CAAC,IAAI,KAAK,QAAQ,IAAI,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS,CAAA;YACpJ,IAAI,SAAS;gBAAE,OAAO,CAAC,IAAI,CAAC,SAAS,CAAC,CAAA;;gBACjC,WAAW,IAAI,CAAC,CAAA;QACvB,CAAC;IACH,CAAC;IACD,OAAO,EAAE,OAAO,EAAE,CAAC,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,CAAC,EAAE,WAAW,EAAE,CAAA;AACxD,CAAC;AAED,kGAAkG;AAClG,SAAgB,SAAS,CAAC,IAAY,EAAE,QAAkB,EAAE,MAAM,GAAG,qBAAqB,EAAE,UAAU,GAAG,mBAAmB;IAC1H,MAAM,KAAK,GAAG,IAAI,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,MAAM,OAAO,GAAG,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,IAAI,CAAC,CAAA;IACrF,MAAM,KAAK,GAA4B,EAAE,CAAA;IACzC,IAAI,OAAO,GAAG,CAAC,CAAA;IACf,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;QACzC,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,KAAK,CAAC;YAAE,MAAK;QAChC,OAAO,IAAI,CAAC,CAAA;QACZ,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,KAAK,GAAG,MAAM,CAAC,CAAA;QAC/C,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,EAAE,KAAK,CAAC,KAAK,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,GAAG,MAAM,CAAC,CAAA;QACzE,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAA;QACpC,IAAI,IAAI,IAAI,KAAK,IAAI,IAAI,CAAC,CAAC,CAAC;YAAE,IAAI,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC,CAAA;;YACzD,KAAK,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,CAAA;IAC/B,CAAC;IACD,MAAM,QAAQ,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,UAAU,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,EAAE,GAAG,CAAC,EAAE,EAAE,CAAC,UAAU,KAAK,IAAI,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC,EAAE,CAAC,CAAA;IACvH,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAA;AAC9B,CAAC;AAED,SAAS,QAAQ,CAAC,IAAY;IAC5B,MAAM,MAAM,GAAa,EAAE,CAAA;IAC3B,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,IAAI,CAAC,MAAM,EAAE,KAAK,IAAI,8BAAsB;QAAE,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,KAAK,GAAG,8BAAsB,CAAC,CAAC,CAAA;IACxI,OAAO,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAA;AACtC,CAAC;AAED,8GAA8G;AAC9G,SAAgB,YAAY,CAAC,GAA4B,EAAE,MAAgB,EAAE,OAAkC;IAC7G,MAAM,GAAG,GAA4B,EAAE,GAAG,GAAG,EAAE,CAAA;IAC/C,KAAK,MAAM,KAAK,IAAI,MAAM;QAAE,IAAI,CAAC,CAAC,KAAK,IAAI,GAAG,CAAC;YAAE,GAAG,CAAC,KAAK,CAAC,GAAG,IAAI,CAAA;IAClE,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC,OAAO,aAAP,OAAO,cAAP,OAAO,GAAI,EAAE,CAAC;SAC1C,MAAM,CAAC,CAAC,CAAC,KAAK,EAAE,MAAM,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,IAAI,IAAI,GAAG,CAAC,KAAK,CAAC,KAAK,SAAS,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;SACpH,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,CAAA;IAC1B,IAAI,OAAO,CAAC,MAAM;QAAE,GAAG,CAAC,OAAO,GAAG,OAAO,CAAA;IACzC,OAAO,GAAG,CAAA;AACZ,CAAC;AAED,SAAS,WAAW,CAAC,IAAY,EAAE,IAAa;IAC9C,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,aAAa,CAAC,CAAA;IACvC,IAAI,CAAC,KAAK;QAAE,MAAM,IAAI,KAAK,CAAC,4CAA4C,CAAC,CAAA;IACzE,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAA4B,CAAA;IAC9D,IAAI,CAAC,IAAI;QAAE,OAAO,CAAC,MAAM,CAAC,CAAA;IAC1B,OAAO,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAE,MAAM,CAAC,IAAkC,CAAC,CAAC,CAAC,EAAE,CAAA;AACrF,CAAC;AAED,KAAK,UAAU,WAAW,CAAC,MAAc;IACvC,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,MAAM,aAAE,CAAC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;QAC7C,OAAO,IAAI,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,MAAM,CAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAA0B,CAAC,MAAM,CAAC,CAAC,CAAC,CAAA;IAC1H,CAAC;IAAC,WAAM,CAAC;QACP,OAAO,IAAI,GAAG,EAAE,CAAA;IAClB,CAAC;AACH,CAAC;AAEM,KAAK,UAAU,gBAAgB,CAAC,OAA0B,EAAE,IAAoB;;IACrF,IAAI,CAAC,CAAA,MAAA,OAAO,CAAC,KAAK,0CAAE,MAAM,CAAA,IAAI,CAAC,OAAO,CAAC,QAAQ;QAAE,MAAM,IAAI,KAAK,CAAC,2BAA2B,CAAC,CAAA;IAC7F,MAAM,EAAE,OAAO,EAAE,WAAW,EAAE,GAAG,MAAM,SAAS,CAAC,IAAI,CAAC,YAAY,EAAE,OAAO,CAAC,CAAA;IAC5E,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,YAAY,EAAE,OAAO,CAAC,UAAU,CAAC,CAAA;IAC5D,MAAM,OAAO,GAAG,MAAM,WAAW,CAAC,MAAM,CAAC,CAAA;IACzC,MAAM,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAA;IAChE,MAAM,IAAI,GAAG,MAAA,OAAO,CAAC,IAAI,mCAAI,KAAK,CAAA;IAClC,MAAM,OAAO,GAAG,KAAK,EAAE,MAAc,EAAE,EAAE;;QACvC,MAAM,IAAI,GAAG,MAAM,aAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,YAAY,EAAE,MAAM,CAAC,EAAE,MAAM,CAAC,CAAA;QACzE,IAAI,CAAC,CAAA,MAAA,OAAO,CAAC,OAAO,0CAAE,QAAQ,CAAC,MAAM,CAAA;YAAE,OAAO,EAAE,KAAK,EAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,SAA+B,EAAE,CAAA;QACjH,MAAM,EAAE,QAAQ,EAAE,OAAO,EAAE,GAAG,SAAS,CAAC,IAAI,EAAE,OAAO,CAAC,OAAO,CAAC,QAAQ,EAAE,OAAO,CAAC,OAAO,CAAC,MAAM,EAAE,OAAO,CAAC,OAAO,CAAC,UAAU,CAAC,CAAA;QAC3H,OAAO,EAAE,KAAK,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,OAAO,EAAE,CAAA;IAC5E,CAAC,CAAA;IAED,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;QACnB,IAAI,KAAK,GAAG,CAAC,CAAA;QACb,IAAI,KAAK,GAAG,CAAC,CAAA;QACb,IAAI,OAAO,GAAG,CAAC,CAAA;QACf,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;YAC7B,MAAM,EAAE,KAAK,EAAE,GAAG,MAAM,OAAO,CAAC,MAAM,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,CAAC,EAAE,KAAK,EAAE,EAAc,EAAE,CAAC,CAAC,CAAA;YAChF,IAAI,CAAC,KAAK,CAAC,MAAM;gBAAE,OAAO,IAAI,CAAC,CAAA;YAC/B,KAAK,IAAI,KAAK,CAAC,MAAM,CAAA;YACrB,KAAK,IAAI,KAAK,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,CAAC,GAAG,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC,CAAA;QAC5D,CAAC;QACD,OAAO;YACL,MAAM,EAAE,IAAI;YACZ,SAAS,EAAE,OAAO,CAAC,MAAM;YACzB,WAAW,EAAE,OAAO,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM;YAC5C,uBAAuB,EAAE,WAAW;YACpC,OAAO;YACP,KAAK;YACL,UAAU,EAAE,KAAK;YACjB,eAAe,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,GAAG,CAAC,CAAC,GAAG,KAAK,GAAG,CAAC,oBAAoB,GAAG,aAAa,CAAC;YACvF,KAAK,EAAE,IAAI,CAAC,OAAO;SACpB,CAAA;IACH,CAAC;IAED,MAAM,aAAE,CAAC,KAAK,CAAC,cAAI,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;IACzD,MAAM,QAAQ,GAA4C,EAAE,CAAA;IAC5D,IAAI,KAAK,GAAG,CAAC,CAAA;IACb,IAAI,IAAI,GAAG,CAAC,CAAA;IACZ,IAAI,KAAK,GAAG,CAAC,CAAA;IACb,IAAI,OAAO,GAAG,CAAC,CAAA;IACf,IAAI,UAAU,GAAG,CAAC,CAAA;IAClB,IAAI,IAAI,GAAG,CAAC,CAAA;IACZ,MAAM,MAAM,GAAG,KAAK,EAAE,GAA4B,EAAE,EAAE;QACpD,MAAM,aAAE,CAAC,UAAU,CAAC,MAAM,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;QACvD,IAAI,IAAI,CAAC,CAAA;QACT,IAAI,GAAG,CAAC,KAAK,KAAK,IAAI;YAAE,KAAK,IAAI,CAAC,CAAA;IACpC,CAAC,CAAA;IAED,MAAM,KAAK,GAAG,CAAC,GAAG,OAAO,CAAC,CAAA;IAC1B,MAAM,MAAM,GAAG,KAAK,IAAI,EAAE;;QACxB,OAAO,KAAK,CAAC,MAAM,EAAE,CAAC;YACpB,IAAI,MAAA,IAAI,CAAC,MAAM,0CAAE,OAAO;gBAAE,OAAM;YAChC,MAAM,MAAM,GAAG,KAAK,CAAC,KAAK,EAAY,CAAA;YACtC,IAAI,CAAC;gBACH,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,GAAG,MAAM,OAAO,CAAC,MAAM,CAAC,CAAA;gBAChD,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC;oBAClB,0FAA0F;oBAC1F,OAAO,IAAI,CAAC,CAAA;oBACZ,MAAM,MAAM,CAAC,YAAY,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,kBAAkB,EAAE,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,CAAA;oBAC7G,SAAQ;gBACV,CAAC;gBACD,MAAM,OAAO,GAA8B,EAAE,CAAA;gBAC7C,KAAK,MAAM,CAAC,KAAK,EAAE,IAAI,CAAC,IAAI,KAAK,CAAC,OAAO,EAAE,EAAE,CAAC;oBAC5C,KAAK,IAAI,CAAC,CAAA;oBACV,UAAU,IAAI,IAAI,CAAC,MAAM,CAAA;oBACzB,MAAM,MAAM,GAAG,MAAM,IAAA,iBAAY,EAAC;wBAChC,KAAK,EAAE,IAAI,CAAC,KAAK;wBACjB,WAAW,EAAE,IAAI,CAAC,MAAM;wBACxB,MAAM,EAAE,IAAI,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,UAAU;wBACvC,MAAM,EAAE,aAAa,OAAO,CAAC,QAAQ,aAAa,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,GACzE,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,qBAAqB,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC,EAC7E,iBAAiB,MAAM,GAAG,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,UAAU,KAAK,GAAG,CAAC,OAAO,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,GAAG,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,KAAK,OAAO,oBAAoB,CAAC,CAAC,CAAC,EAAE,OAAO,IAAI,EAAE;qBACxK,CAAC,CAAA;oBACF,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAAA;oBAC1B,IAAI,KAAK,EAAE,CAAC;wBACV,MAAM,WAAW,GAAG,MAAA,KAAK,CAAC,WAAW,mCAAI,CAAC,CAAA;wBAC1C,MAAM,YAAY,GAAG,MAAA,KAAK,CAAC,YAAY,mCAAI,CAAC,CAAA;wBAC5C,IAAI,CAAC,WAAW,CAAC;4BACf,WAAW;4BACX,YAAY;4BACZ,WAAW,EAAE,MAAA,KAAK,CAAC,WAAW,mCAAI,WAAW,GAAG,YAAY;4BAC5D,eAAe,EAAE,MAAA,KAAK,CAAC,iBAAiB,0CAAE,eAAe;4BACzD,gBAAgB,EAAE,MAAA,KAAK,CAAC,iBAAiB,0CAAE,gBAAgB;yBAC5D,CAAC,CAAA;oBACJ,CAAC;oBACD,KAAK,MAAM,GAAG,IAAI,WAAW,CAAC,MAAM,CAAC,IAAI,EAAE,IAAI,CAAC;wBAAE,OAAO,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,GAAG,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,KAAK,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,GAAG,GAAG,EAAE,CAAC,CAAA;gBACtI,CAAC;gBACD,0FAA0F;gBAC1F,yFAAyF;gBACzF,kEAAkE;gBAClE,MAAM,IAAI,GAAG,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,MAAA,OAAO,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,KAAK,KAAK,IAAI,CAAC,mCAAI,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAA;gBACvG,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAA;gBAC1E,KAAK,MAAM,GAAG,IAAI,KAAK;oBAAE,MAAM,MAAM,CAAC,YAAY,CAAC,GAAG,EAAE,OAAO,CAAC,MAAM,EAAE,OAAO,CAAC,OAAO,CAAC,CAAC,CAAA;YAC3F,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAG,KAAe,CAAC,OAAO,EAAE,CAAC,CAAA;YACnE,CAAC;oBAAS,CAAC;gBACT,IAAI,IAAI,CAAC,CAAA;gBACT,IAAI,IAAI,GAAG,CAAC,KAAK,CAAC,IAAI,IAAI,KAAK,OAAO,CAAC,MAAM;oBAAE,MAAA,IAAI,CAAC,UAAU,qDAAG,IAAI,EAAE,OAAO,CAAC,MAAM,CAAC,CAAA;YACxF,CAAC;QACH,CAAC;IACH,CAAC,CAAA;IACD,MAAA,IAAI,CAAC,UAAU,qDAAG,CAAC,EAAE,OAAO,CAAC,MAAM,CAAC,CAAA;IACpC,MAAM,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,MAAA,OAAO,CAAC,WAAW,mCAAI,mBAAmB,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC,CAAA;IAEpI,OAAO;QACL,UAAU,EAAE,OAAO,CAAC,UAAU;QAC9B,SAAS,EAAE,OAAO,CAAC,MAAM;QACzB,WAAW,EAAE,OAAO,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM;QAC5C,uBAAuB,EAAE,WAAW;QACpC,OAAO;QACP,KAAK;QACL,IAAI;QACJ,KAAK;QACL,MAAM,EAAE,QAAQ,CAAC,MAAM;QACvB,QAAQ,EAAE,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC;QAC9B,UAAU;QACV,KAAK,EAAE,IAAI,CAAC,OAAO;QACnB,IAAI,EAAE,yFAAyF;KAChG,CAAA;AACH,CAAC"}

package/dist/research/extract-texts.test.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=extract-texts.test.d.ts.map

package/dist/research/extract-texts.test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"extract-texts.test.d.ts","sourceRoot":"","sources":["../../src/research/extract-texts.test.ts"],"names":[],"mappings":""}

package/dist/research/extract-texts.test.js

@@ -0,0 +1,120 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const strict_1 = __importDefault(require("node:assert/strict"));
+const node_fs_1 = require("node:fs");
+const node_os_1 = __importDefault(require("node:os"));
+const node_path_1 = __importDefault(require("node:path"));
+const node_test_1 = require("node:test");
+const test_1 = require("ai/test");
+const extract_texts_1 = require("./extract-texts");
+function answer(text) {
+    return {
+        content: [{ type: 'text', text }],
+        finishReason: { unified: 'stop', raw: 'stop' },
+        usage: {
+            inputTokens: { total: 100, noCache: 20, cacheRead: 80, cacheWrite: 0 },
+            outputTokens: { total: 10, text: 10, reasoning: 0 },
+        },
+        warnings: [],
+    };
+}
+async function workspace(files) {
+    const dir = await node_fs_1.promises.mkdtemp(node_path_1.default.join(node_os_1.default.tmpdir(), 'extract-'));
+    for (const [name, content] of Object.entries(files)) {
+        await node_fs_1.promises.mkdir(node_path_1.default.dirname(node_path_1.default.join(dir, name)), { recursive: true });
+        await node_fs_1.promises.writeFile(node_path_1.default.join(dir, name), content);
+    }
+    return dir;
+}
+(0, node_test_1.describe)('windowsOf', () => {
+    (0, node_test_1.it)('merges overlapping passages and labels their offsets', () => {
+        const text = `${'x'.repeat(5000)}cancel within 30 days${'y'.repeat(100)}nonrenewal 45 days${'z'.repeat(5000)}`;
+        const { passages, matches } = (0, extract_texts_1.windowsOf)(text, ['cancel', 'non-?renew'], 500);
+        strict_1.default.equal(matches, 2);
+        strict_1.default.equal(passages.length, 1);
+        strict_1.default.match(passages[0], /^\[chars 4500–/);
+        strict_1.default.match(passages[0], /cancel within 30 days[\s\S]*nonrenewal 45 days/);
+    });
+    (0, node_test_1.it)('finds nothing in a document without a match', () => {
+        strict_1.default.deepEqual((0, extract_texts_1.windowsOf)('homeowners policy', ['water shut-?off']).passages, []);
+    });
+});
+(0, node_test_1.describe)('normalizeRow', () => {
+    (0, node_test_1.it)('fills missing fields and flags values outside the codebook', () => {
+        const row = (0, extract_texts_1.normalizeRow)({ category: 'surcharge', credit: 5 }, ['category', 'credit', 'appliesTo'], { category: ['credit', 'eligibility'] });
+        strict_1.default.equal(row.appliesTo, null);
+        strict_1.default.deepEqual(row.invalid, ['category']);
+    });
+});
+(0, node_test_1.describe)('extractFromTexts', () => {
+    (0, node_test_1.it)('sends only the windows, skips documents without a match and resumes', async () => {
+        const dir = await workspace({
+            'data/text/a.txt': `${'filler '.repeat(20000)}Notice of cancellation: 30 days. Nonrenewal: 45 days.`,
+            'data/text/b.txt': 'Nothing relevant here.',
+            'data/manifest.jsonl': '{"textPath":"data/text/a.txt"}\n{"textPath":"data/text/b.txt"}\n{"name":"scan.pdf"}\n',
+        });
+        const prompts = [];
+        const model = new test_1.MockLanguageModelV3({
+            doGenerate: async (options) => {
+                prompts.push(JSON.stringify(options.prompt));
+                return answer('{"rows":[{"noticeType":"cancellation","days":30,"quote":"30 days","found":true},{"noticeType":"nonrenewal","days":45,"quote":"45 days","found":true}]}');
+            },
+        });
+        const usage = [];
+        const request = {
+            manifest: 'data/manifest.jsonl',
+            question: 'Notice periods',
+            fields: ['noticeType', 'days'],
+            outputPath: 'data/extracted/notices.jsonl',
+            windows: { patterns: ['cancel', 'non-?renew'] },
+            many: true,
+        };
+        const deps = { workspaceDir: dir, model, modelId: 'mock', recordUsage: (u) => usage.push(u.totalTokens) };
+        const result = await (0, extract_texts_1.extractFromTexts)(request, deps);
+        strict_1.default.equal(result.documents, 2);
+        strict_1.default.equal(result.manifestRowsWithoutText, 1);
+        strict_1.default.equal(result.noMatch, 1);
+        strict_1.default.equal(result.calls, 1);
+        strict_1.default.equal(result.found, 2);
+        strict_1.default.ok(prompts[0].length < 10000, 'only the passages reach the model');
+        strict_1.default.deepEqual(usage, [110]);
+        const rows = (await node_fs_1.promises.readFile(node_path_1.default.join(dir, 'data/extracted/notices.jsonl'), 'utf8')).trim().split('\n').map((line) => JSON.parse(line));
+        strict_1.default.deepEqual(rows.map((row) => { var _a; return [row.source, (_a = row.days) !== null && _a !== void 0 ? _a : null]; }).sort(), [['data/text/a.txt', 30], ['data/text/a.txt', 45], ['data/text/b.txt', null]]);
+        const again = await (0, extract_texts_1.extractFromTexts)(request, deps);
+        strict_1.default.equal(again.documents, 0);
+        strict_1.default.equal(again.alreadyDone, 2);
+        strict_1.default.equal(prompts.length, 1);
+    });
+    (0, node_test_1.it)('reads a long document in chunks instead of cutting it, and dryRun calls no model', async () => {
+        const dir = await workspace({ 'data/text/manual.txt': `${'a'.repeat(extract_texts_1.EXTRACTION_CHUNK_CHARS)}water shut-off credit 5%` });
+        let calls = 0;
+        const model = new test_1.MockLanguageModelV3({
+            doGenerate: async (options) => {
+                calls += 1;
+                const found = JSON.stringify(options.prompt).includes('credit 5%');
+                return answer(found ? '{"credit":"5%","quote":"credit 5%","found":true}' : '{"credit":null,"quote":null,"found":false}');
+            },
+        });
+        const request = { paths: ['data/text/manual.txt'], question: 'Water device credit', fields: ['credit'], outputPath: 'data/out.jsonl' };
+        const deps = { workspaceDir: dir, model, modelId: 'mock', recordUsage: () => undefined };
+        const estimate = await (0, extract_texts_1.extractFromTexts)({ ...request, dryRun: true }, deps);
+        strict_1.default.equal(estimate.calls, 2);
+        strict_1.default.equal(calls, 0);
+        const result = await (0, extract_texts_1.extractFromTexts)(request, deps);
+        strict_1.default.equal(result.calls, 2);
+        const [row] = (await node_fs_1.promises.readFile(node_path_1.default.join(dir, 'data/out.jsonl'), 'utf8')).trim().split('\n').map((line) => JSON.parse(line));
+        strict_1.default.equal(row.credit, '5%');
+        strict_1.default.equal(row.part, 2);
+    });
+    (0, node_test_1.it)('keeps the fields the model filled when it answers found=false', async () => {
+        const dir = await workspace({ 'data/text/form.txt': 'GL 4065 new endorsement text' });
+        const model = new test_1.MockLanguageModelV3({ doGenerate: async () => answer('{"purpose":"Adds a new exclusion","quote":null,"found":false}') });
+        await (0, extract_texts_1.extractFromTexts)({ paths: ['data/text/form.txt'], question: 'Summarize', fields: ['purpose'], outputPath: 'data/out.jsonl' }, { workspaceDir: dir, model, modelId: 'mock', recordUsage: () => undefined });
+        const [row] = (await node_fs_1.promises.readFile(node_path_1.default.join(dir, 'data/out.jsonl'), 'utf8')).trim().split('\n').map((line) => JSON.parse(line));
+        strict_1.default.equal(row.purpose, 'Adds a new exclusion');
+    });
+});
+//# sourceMappingURL=extract-texts.test.js.map

package/dist/research/extract-texts.test.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"extract-texts.test.js","sourceRoot":"","sources":["../../src/research/extract-texts.test.ts"],"names":[],"mappings":";;;;;AAAA,gEAAuC;AACvC,qCAAwC;AACxC,sDAAwB;AACxB,0DAA4B;AAC5B,yCAAwC;AACxC,kCAA6C;AAC7C,mDAAmG;AAEnG,SAAS,MAAM,CAAC,IAAY;IAC1B,OAAO;QACL,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAe,EAAE,IAAI,EAAE,CAAC;QAC1C,YAAY,EAAE,EAAE,OAAO,EAAE,MAAe,EAAE,GAAG,EAAE,MAAM,EAAE;QACvD,KAAK,EAAE;YACL,WAAW,EAAE,EAAE,KAAK,EAAE,GAAG,EAAE,OAAO,EAAE,EAAE,EAAE,SAAS,EAAE,EAAE,EAAE,UAAU,EAAE,CAAC,EAAE;YACtE,YAAY,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,IAAI,EAAE,EAAE,EAAE,SAAS,EAAE,CAAC,EAAE;SACpD;QACD,QAAQ,EAAE,EAAE;KACb,CAAA;AACH,CAAC;AAED,KAAK,UAAU,SAAS,CAAC,KAA6B;IACpD,MAAM,GAAG,GAAG,MAAM,kBAAE,CAAC,OAAO,CAAC,mBAAI,CAAC,IAAI,CAAC,iBAAE,CAAC,MAAM,EAAE,EAAE,UAAU,CAAC,CAAC,CAAA;IAChE,KAAK,MAAM,CAAC,IAAI,EAAE,OAAO,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;QACpD,MAAM,kBAAE,CAAC,KAAK,CAAC,mBAAI,CAAC,OAAO,CAAC,mBAAI,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;QACvE,MAAM,kBAAE,CAAC,SAAS,CAAC,mBAAI,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,EAAE,OAAO,CAAC,CAAA;IACnD,CAAC;IACD,OAAO,GAAG,CAAA;AACZ,CAAC;AAED,IAAA,oBAAQ,EAAC,WAAW,EAAE,GAAG,EAAE;IACzB,IAAA,cAAE,EAAC,sDAAsD,EAAE,GAAG,EAAE;QAC9D,MAAM,IAAI,GAAG,GAAG,GAAG,CAAC,MAAM,CAAC,IAAK,CAAC,wBAAwB,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,qBAAqB,GAAG,CAAC,MAAM,CAAC,IAAK,CAAC,EAAE,CAAA;QAChH,MAAM,EAAE,QAAQ,EAAE,OAAO,EAAE,GAAG,IAAA,yBAAS,EAAC,IAAI,EAAE,CAAC,QAAQ,EAAE,YAAY,CAAC,EAAE,GAAG,CAAC,CAAA;QAC5E,gBAAM,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC,CAAA;QACxB,gBAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC,CAAA;QAChC,gBAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,gBAAgB,CAAC,CAAA;QAC3C,gBAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,gDAAgD,CAAC,CAAA;IAC7E,CAAC,CAAC,CAAA;IAEF,IAAA,cAAE,EAAC,6CAA6C,EAAE,GAAG,EAAE;QACrD,gBAAM,CAAC,SAAS,CAAC,IAAA,yBAAS,EAAC,mBAAmB,EAAE,CAAC,iBAAiB,CAAC,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAA;IACpF,CAAC,CAAC,CAAA;AACJ,CAAC,CAAC,CAAA;AAEF,IAAA,oBAAQ,EAAC,cAAc,EAAE,GAAG,EAAE;IAC5B,IAAA,cAAE,EAAC,4DAA4D,EAAE,GAAG,EAAE;QACpE,MAAM,GAAG,GAAG,IAAA,4BAAY,EAAC,EAAE,QAAQ,EAAE,WAAW,EAAE,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC,UAAU,EAAE,QAAQ,EAAE,WAAW,CAAC,EAAE,EAAE,QAAQ,EAAE,CAAC,QAAQ,EAAE,aAAa,CAAC,EAAE,CAAC,CAAA;QAC5I,gBAAM,CAAC,KAAK,CAAC,GAAG,CAAC,SAAS,EAAE,IAAI,CAAC,CAAA;QACjC,gBAAM,CAAC,SAAS,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC,UAAU,CAAC,CAAC,CAAA;IAC7C,CAAC,CAAC,CAAA;AACJ,CAAC,CAAC,CAAA;AAEF,IAAA,oBAAQ,EAAC,kBAAkB,EAAE,GAAG,EAAE;IAChC,IAAA,cAAE,EAAC,qEAAqE,EAAE,KAAK,IAAI,EAAE;QACnF,MAAM,GAAG,GAAG,MAAM,SAAS,CAAC;YAC1B,iBAAiB,EAAE,GAAG,SAAS,CAAC,MAAM,CAAC,KAAM,CAAC,uDAAuD;YACrG,iBAAiB,EAAE,wBAAwB;YAC3C,qBAAqB,EAAE,uFAAuF;SAC/G,CAAC,CAAA;QACF,MAAM,OAAO,GAAa,EAAE,CAAA;QAC5B,MAAM,KAAK,GAAG,IAAI,0BAAmB,CAAC;YACpC,UAAU,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE;gBAC5B,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAA;gBAC5C,OAAO,MAAM,CAAC,wJAAwJ,CAAC,CAAA;YACzK,CAAC;SACF,CAAC,CAAA;QACF,MAAM,KAAK,GAAa,EAAE,CAAA;QAC1B,MAAM,OAAO,GAAG;YACd,QAAQ,EAAE,qBAAqB;YAC/B,QAAQ,EAAE,gBAAgB;YAC1B,MAAM,EAAE,CAAC,YAAY,EAAE,MAAM,CAAC;YAC9B,UAAU,EAAE,8BAA8B;YAC1C,OAAO,EAAE,EAAE,QAAQ,EAAE,CAAC,QAAQ,EAAE,YAAY,CAAC,EAAE;YAC/C,IAAI,EAAE,IAAI;SACX,CAAA;QACD,MAAM,IAAI,GAAG,EAAE,YAAY,EAAE,GAAG,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,WAAW,EAAE,CAAC,CAA0B,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,WAAW,CAAC,EAAE,CAAA;QAClI,MAAM,MAAM,GAAG,MAAM,IAAA,gCAAgB,EAAC,OAAO,EAAE,IAAI,CAAC,CAAA;QAEpD,gBAAM,CAAC,KAAK,CAAC,MAAM,CAAC,SAAS,EAAE,CAAC,CAAC,CAAA;QACjC,gBAAM,CAAC,KAAK,CAAC,MAAM,CAAC,uBAAuB,EAAE,CAAC,CAAC,CAAA;QAC/C,gBAAM,CAAC,KAAK,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC,CAAC,CAAA;QAC/B,gBAAM,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC,CAAA;QAC7B,gBAAM,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC,CAAA;QAC7B,gBAAM,CAAC,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,MAAM,GAAG,KAAM,EAAE,mCAAmC,CAAC,CAAA;QAC1E,gBAAM,CAAC,SAAS,CAAC,KAAK,EAAE,CAAC,GAAG,CAAC,CAAC,CAAA;QAC9B,MAAM,IAAI,GAAG,CAAC,MAAM,kBAAE,CAAC,QAAQ,CAAC,mBAAI,CAAC,IAAI,CAAC,GAAG,EAAE,8BAA8B,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAA;QAC3I,gBAAM,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,WAAC,OAAA,CAAC,GAAG,CAAC,MAAM,EAAE,MAAA,GAAG,CAAC,IAAI,mCAAI,IAAI,CAAC,CAAA,EAAA,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,iBAAiB,EAAE,EAAE,CAAC,EAAE,CAAC,iBAAiB,EAAE,EAAE,CAAC,EAAE,CAAC,iBAAiB,EAAE,IAAI,CAAC,CAAC,CAAC,CAAA;QAEzJ,MAAM,KAAK,GAAG,MAAM,IAAA,gCAAgB,EAAC,OAAO,EAAE,IAAI,CAAC,CAAA;QACnD,gBAAM,CAAC,KAAK,CAAC,KAAK,CAAC,SAAS,EAAE,CAAC,CAAC,CAAA;QAChC,gBAAM,CAAC,KAAK,CAAC,KAAK,CAAC,WAAW,EAAE,CAAC,CAAC,CAAA;QAClC,gBAAM,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAA;IACjC,CAAC,CAAC,CAAA;IAEF,IAAA,cAAE,EAAC,kFAAkF,EAAE,KAAK,IAAI,EAAE;QAChG,MAAM,GAAG,GAAG,MAAM,SAAS,CAAC,EAAE,sBAAsB,EAAE,GAAG,GAAG,CAAC,MAAM,CAAC,sCAAsB,CAAC,0BAA0B,EAAE,CAAC,CAAA;QACxH,IAAI,KAAK,GAAG,CAAC,CAAA;QACb,MAAM,KAAK,GAAG,IAAI,0BAAmB,CAAC;YACpC,UAAU,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE;gBAC5B,KAAK,IAAI,CAAC,CAAA;gBACV,MAAM,KAAK,GAAG,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAA;gBAClE,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,kDAAkD,CAAC,CAAC,CAAC,4CAA4C,CAAC,CAAA;YAC1H,CAAC;SACF,CAAC,CAAA;QACF,MAAM,OAAO,GAAG,EAAE,KAAK,EAAE,CAAC,sBAAsB,CAAC,EAAE,QAAQ,EAAE,qBAAqB,EAAE,MAAM,EAAE,CAAC,QAAQ,CAAC,EAAE,UAAU,EAAE,gBAAgB,EAAE,CAAA;QACtI,MAAM,IAAI,GAAG,EAAE,YAAY,EAAE,GAAG,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,WAAW,EAAE,GAAG,EAAE,CAAC,SAAS,EAAE,CAAA;QAExF,MAAM,QAAQ,GAAG,MAAM,IAAA,gCAAgB,EAAC,EAAE,GAAG,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,EAAE,IAAI,CAAC,CAAA;QAC3E,gBAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,KAAK,EAAE,CAAC,CAAC,CAAA;QAC/B,gBAAM,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC,CAAC,CAAA;QAEtB,MAAM,MAAM,GAAG,MAAM,IAAA,gCAAgB,EAAC,OAAO,EAAE,IAAI,CAAC,CAAA;QACpD,gBAAM,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC,CAAA;QAC7B,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,kBAAE,CAAC,QAAQ,CAAC,mBAAI,CAAC,IAAI,CAAC,GAAG,EAAE,gBAAgB,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAA;QAC9H,gBAAM,CAAC,KAAK,CAAC,GAAG,CAAC,MAAM,EAAE,IAAI,CAAC,CAAA;QAC9B,gBAAM,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC,CAAA;IAC3B,CAAC,CAAC,CAAA;IAEF,IAAA,cAAE,EAAC,+DAA+D,EAAE,KAAK,IAAI,EAAE;QAC7E,MAAM,GAAG,GAAG,MAAM,SAAS,CAAC,EAAE,oBAAoB,EAAE,8BAA8B,EAAE,CAAC,CAAA;QACrF,MAAM,KAAK,GAAG,IAAI,0BAAmB,CAAC,EAAE,UAAU,EAAE,KAAK,IAAI,EAAE,CAAC,MAAM,CAAC,+DAA+D,CAAC,EAAE,CAAC,CAAA;QAC1I,MAAM,IAAA,gCAAgB,EACpB,EAAE,KAAK,EAAE,CAAC,oBAAoB,CAAC,EAAE,QAAQ,EAAE,WAAW,EAAE,MAAM,EAAE,CAAC,SAAS,CAAC,EAAE,UAAU,EAAE,gBAAgB,EAAE,EAC3G,EAAE,YAAY,EAAE,GAAG,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,WAAW,EAAE,GAAG,EAAE,CAAC,SAAS,EAAE,CAC5E,CAAA;QACD,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,kBAAE,CAAC,QAAQ,CAAC,mBAAI,CAAC,IAAI,CAAC,GAAG,EAAE,gBAAgB,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAA;QAC9H,gBAAM,CAAC,KAAK,CAAC,GAAG,CAAC,OAAO,EAAE,sBAAsB,CAAC,CAAA;IACnD,CAAC,CAAC,CAAA;AACJ,CAAC,CAAC,CAAA"}

package/dist/research/extraction-tool.d.ts

@@ -1,6 +1,6 @@
 import { type Tool } from 'ai';
 import type { AgentModel } from '../agent/definition';
 import type { SessionToolContext } from '../session/tool-context';
-/** Bulk extraction on the cheaper model: one call per file, structured rows appended to a JSONL file. */
+/** Bulk extraction on the cheaper model (extract-texts.ts): rows go to a JSONL file, the conversation gets counts. */
 export declare function createExtractionTool(context: SessionToolContext, modelConfig: AgentModel): Tool;
 //# sourceMappingURL=extraction-tool.d.ts.map

package/dist/research/extraction-tool.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"extraction-tool.d.ts","sourceRoot":"","sources":["../../src/research/extraction-tool.ts"],"names":[],"mappings":"AAEA,OAAO,EAAsB,KAAK,IAAI,EAAE,MAAM,IAAI,CAAA;AAGlD,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAA;AACrD,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,yBAAyB,CAAA;AAKjE,yGAAyG;AACzG,wBAAgB,oBAAoB,CAAC,OAAO,EAAE,kBAAkB,EAAE,WAAW,EAAE,UAAU,GAAG,IAAI,CAiD/F"}
+{"version":3,"file":"extraction-tool.d.ts","sourceRoot":"","sources":["../../src/research/extraction-tool.ts"],"names":[],"mappings":"AAAA,OAAO,EAAQ,KAAK,IAAI,EAAE,MAAM,IAAI,CAAA;AAGpC,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAA;AACrD,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,yBAAyB,CAAA;AAGjE,sHAAsH;AACtH,wBAAgB,oBAAoB,CAAC,OAAO,EAAE,kBAAkB,EAAE,WAAW,EAAE,UAAU,GAAG,IAAI,CAkC/F"}

package/dist/research/extraction-tool.js

@@ -1,67 +1,41 @@
 "use strict";
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.createExtractionTool = createExtractionTool;
-const node_fs_1 = require("node:fs");
-const node_path_1 = __importDefault(require("node:path"));
 const ai_1 = require("ai");
 const zod_1 = require("zod");
 const resolve_model_1 = require("../models/resolve-model");
-const usage_1 = require("../models/usage");
-const MAX_EXTRACTION_CHARS = 60000;
-/** Bulk extraction on the cheaper model: one call per file, structured rows appended to a JSONL file. */
+const extract_texts_1 = require("./extract-texts");
+/** Bulk extraction on the cheaper model (extract-texts.ts): rows go to a JSONL file, the conversation gets counts. */
 function createExtractionTool(context, modelConfig) {
     const { workspaceDir, abortSignal, recordUsage } = context;
     return (0, ai_1.tool)({
-        description: 'Extract the same facts from many extracted-text files with a cheaper model, one file at a time, appending one JSON row per file to outputPath (JSONL). Use after search_documents/download_attachment saved texts under data/text/. Each row has the requested fields plus `quote` (the supporting span) and `source`.',
+        description: `Ask the same question of many extracted-text files with a cheaper model, writing JSON rows to outputPath (JSONL); you get counts back, never the documents. Input: a manifest (fetch_attachments outputManifest or a saveHits file) or paths. Use windows.patterns (regexes) whenever you know what the passage looks like: only the text around matches is sent, and documents without a match cost nothing. Without windows each document is read in ${extract_texts_1.EXTRACTION_CHUNK_CHARS.toLocaleString()}-character chunks. Set many=true when one document can state the fact several times (one row per statement). allowed holds codebook values per field; rows outside it are flagged "invalid". Run dryRun first on large sets to see the call count and estimated tokens. Re-running with the same outputPath skips documents already done.`,
         inputSchema: zod_1.z.object({
-            paths: zod_1.z.array(zod_1.z.string().min(1).max(500)).min(1).max(50),
+            paths: zod_1.z.array(zod_1.z.string().min(1).max(500)).min(1).max(200).optional(),
+            manifest: zod_1.z.string().min(1).max(300).optional().describe('JSONL/JSON manifest whose rows carry textPath'),
             question: zod_1.z.string().min(1).max(2000).describe('What to look for, precisely'),
             fields: zod_1.z.array(zod_1.z.string().min(1).max(60)).min(1).max(15).describe('Field names each row must contain'),
             outputPath: zod_1.z.string().min(1).max(300).describe('e.g. data/extracted/water-valve.jsonl'),
+            windows: zod_1.z
+                .object({
+                patterns: zod_1.z.array(zod_1.z.string().min(1).max(200)).min(1).max(20).describe('Case-insensitive regexes, e.g. "cancel", "non-?renew", "water (shut|leak)"'),
+                radius: zod_1.z.number().int().min(200).max(6000).optional().describe('Characters kept on each side of a match (default 1500)'),
+                maxWindows: zod_1.z.number().int().min(1).max(20).optional().describe('Passages per document (default 6)'),
+            })
+                .optional(),
+            many: zod_1.z.boolean().optional(),
+            allowed: zod_1.z.record(zod_1.z.string(), zod_1.z.array(zod_1.z.string().max(100)).max(30)).optional().describe('Codebook: allowed values per field'),
+            concurrency: zod_1.z.number().int().min(1).max(8).optional(),
+            dryRun: zod_1.z.boolean().optional(),
+        }),
+        execute: async (input, { toolCallId }) => (0, extract_texts_1.extractFromTexts)(input, {
+            workspaceDir,
+            model: (0, resolve_model_1.resolveLanguageModel)(modelConfig.provider, modelConfig.id, modelConfig.attribution),
+            modelId: modelConfig.id,
+            signal: abortSignal,
+            recordUsage: (usage) => recordUsage(usage, modelConfig.id),
+            onProgress: (done, total) => context.reportProgress([{ key: 'extract', label: 'Extracting facts', status: done < total ? 'active' : 'done', detail: `${done}/${total}` }], toolCallId),
         }),
-        execute: async ({ paths, question, fields, outputPath }, { toolCallId }) => {
-            const model = (0, resolve_model_1.resolveLanguageModel)(modelConfig.provider, modelConfig.id, modelConfig.attribution);
-            const output = node_path_1.default.resolve(workspaceDir, outputPath);
-            await node_fs_1.promises.mkdir(node_path_1.default.dirname(output), { recursive: true });
-            let rows = 0;
-            const failures = [];
-            const report = (index) => {
-                context.reportProgress([
-                    { key: 'extract', label: 'Extracting facts', status: index < paths.length ? 'active' : 'done', detail: `${index}/${paths.length} documents` },
-                ], toolCallId);
-            };
-            report(0);
-            for (const [index, relative] of paths.entries()) {
-                if (abortSignal.aborted)
-                    break;
-                if (index > 0 && index % 3 === 0)
-                    report(index);
-                try {
-                    const text = (await node_fs_1.promises.readFile(node_path_1.default.resolve(workspaceDir, relative), 'utf8')).slice(0, MAX_EXTRACTION_CHARS);
-                    const result = await (0, ai_1.generateText)({
-                        model,
-                        abortSignal,
-                        system: 'You extract facts from one insurance filing document. Answer with a single JSON object only — no prose. Use null for a field the document does not state. Always include "quote": the exact supporting text (max 300 characters) or null, and "found": true|false.',
-                        prompt: `Question: ${question}\nFields: ${fields.join(', ')}\n\nDocument (${relative}):\n${text}`,
-                    });
-                    const usage = (0, usage_1.usageOf)(result);
-                    if (usage)
-                        recordUsage(usage, modelConfig.id);
-                    const match = result.text.match(/\{[\s\S]*\}/);
-                    const parsed = match ? JSON.parse(match[0]) : { found: false };
-                    await node_fs_1.promises.appendFile(output, `${JSON.stringify({ source: relative, ...parsed })}\n`);
-                    rows += 1;
-                }
-                catch (error) {
-                    failures.push({ path: relative, reason: error.message });
-                }
-            }
-            report(paths.length);
-            return { outputPath, rows, failures, model: modelConfig.id };
-        },
     });
 }
 //# sourceMappingURL=extraction-tool.js.map

package/dist/research/extraction-tool.js.map

@@ -1 +1 @@
-{"version":3,"file":"extraction-tool.js","sourceRoot":"","sources":["../../src/research/extraction-tool.ts"],"names":[],"mappings":";;;;;AAYA,oDAiDC;AA7DD,qCAAwC;AACxC,0DAA4B;AAC5B,2BAAkD;AAClD,6BAAuB;AACvB,2DAA8D;AAG9D,2CAAyC;AAEzC,MAAM,oBAAoB,GAAG,KAAM,CAAA;AAEnC,yGAAyG;AACzG,SAAgB,oBAAoB,CAAC,OAA2B,EAAE,WAAuB;IACvF,MAAM,EAAE,YAAY,EAAE,WAAW,EAAE,WAAW,EAAE,GAAG,OAAO,CAAA;IAC1D,OAAO,IAAA,SAAI,EAAC;QACV,WAAW,EACT,wTAAwT;QAC1T,WAAW,EAAE,OAAC,CAAC,MAAM,CAAC;YACpB,KAAK,EAAE,OAAC,CAAC,KAAK,CAAC,OAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC;YACzD,QAAQ,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,QAAQ,CAAC,6BAA6B,CAAC;YAC7E,MAAM,EAAE,OAAC,CAAC,KAAK,CAAC,OAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,QAAQ,CAAC,mCAAmC,CAAC;YACvG,UAAU,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC,uCAAuC,CAAC;SACzF,CAAC;QACF,OAAO,EAAE,KAAK,EAAE,EAAE,KAAK,EAAE,QAAQ,EAAE,MAAM,EAAE,UAAU,EAAE,EAAE,EAAE,UAAU,EAAE,EAAE,EAAE;YACzE,MAAM,KAAK,GAAG,IAAA,oCAAoB,EAAC,WAAW,CAAC,QAAQ,EAAE,WAAW,CAAC,EAAE,EAAE,WAAW,CAAC,WAAW,CAAC,CAAA;YACjG,MAAM,MAAM,GAAG,mBAAI,CAAC,OAAO,CAAC,YAAY,EAAE,UAAU,CAAC,CAAA;YACrD,MAAM,kBAAE,CAAC,KAAK,CAAC,mBAAI,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;YACzD,IAAI,IAAI,GAAG,CAAC,CAAA;YACZ,MAAM,QAAQ,GAA4C,EAAE,CAAA;YAC5D,MAAM,MAAM,GAAG,CAAC,KAAa,EAAE,EAAE;gBAC/B,OAAO,CAAC,cAAc,CAAC;oBACrB,EAAE,GAAG,EAAE,SAAS,EAAE,KAAK,EAAE,kBAAkB,EAAE,MAAM,EAAE,KAAK,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,MAAM,EAAE,MAAM,EAAE,GAAG,KAAK,IAAI,KAAK,CAAC,MAAM,YAAY,EAAE;iBAC9I,EAAE,UAAU,CAAC,CAAA;YAChB,CAAC,CAAA;YACD,MAAM,CAAC,CAAC,CAAC,CAAA;YACT,KAAK,MAAM,CAAC,KAAK,EAAE,QAAQ,CAAC,IAAI,KAAK,CAAC,OAAO,EAAE,EAAE,CAAC;gBAChD,IAAI,WAAW,CAAC,OAAO;oBAAE,MAAK;gBAC9B,IAAI,KAAK,GAAG,CAAC,IAAI,KAAK,GAAG,CAAC,KAAK,CAAC;oBAAE,MAAM,CAAC,KAAK,CAAC,CAAA;gBAC/C,IAAI,CAAC;oBACH,MAAM,IAAI,GAAG,CAAC,MAAM,kBAAE,CAAC,QAAQ,CAAC,mBAAI,CAAC,OAAO,CAAC,YAAY,EAAE,QAAQ,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,oBAAoB,CAAC,CAAA;oBAC7G,MAAM,MAAM,GAAG,MAAM,IAAA,iBAAY,EAAC;wBAChC,KAAK;wBACL,WAAW;wBACX,MAAM,EACJ,oQAAoQ;wBACtQ,MAAM,EAAE,aAAa,QAAQ,aAAa,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,iBAAiB,QAAQ,OAAO,IAAI,EAAE;qBAClG,CAAC,CAAA;oBACF,MAAM,KAAK,GAAG,IAAA,eAAO,EAAC,MAAM,CAAC,CAAA;oBAC7B,IAAI,KAAK;wBAAE,WAAW,CAAC,KAAK,EAAE,WAAW,CAAC,EAAE,CAAC,CAAA;oBAC7C,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,aAAa,CAAC,CAAA;oBAC9C,MAAM,MAAM,GAAG,KAAK,CAAC,CAAC,CAAE,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAA6B,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,KAAK,EAAE,CAAA;oBAC3F,MAAM,kBAAE,CAAC,UAAU,CAAC,MAAM,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC,EAAE,MAAM,EAAE,QAAQ,EAAE,GAAG,MAAM,EAAE,CAAC,IAAI,CAAC,CAAA;oBACnF,IAAI,IAAI,CAAC,CAAA;gBACX,CAAC;gBAAC,OAAO,KAAK,EAAE,CAAC;oBACf,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAG,KAAe,CAAC,OAAO,EAAE,CAAC,CAAA;gBACrE,CAAC;YACH,CAAC;YACD,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAA;YACpB,OAAO,EAAE,UAAU,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,WAAW,CAAC,EAAE,EAAE,CAAA;QAC9D,CAAC;KACF,CAAC,CAAA;AACJ,CAAC"}
+{"version":3,"file":"extraction-tool.js","sourceRoot":"","sources":["../../src/research/extraction-tool.ts"],"names":[],"mappings":";;AAQA,oDAkCC;AA1CD,2BAAoC;AACpC,6BAAuB;AACvB,2DAA8D;AAG9D,mDAA0E;AAE1E,sHAAsH;AACtH,SAAgB,oBAAoB,CAAC,OAA2B,EAAE,WAAuB;IACvF,MAAM,EAAE,YAAY,EAAE,WAAW,EAAE,WAAW,EAAE,GAAG,OAAO,CAAA;IAC1D,OAAO,IAAA,SAAI,EAAC;QACV,WAAW,EACT,0bAA0b,sCAAsB,CAAC,cAAc,EAAE,2UAA2U;QAC9yB,WAAW,EAAE,OAAC,CAAC,MAAM,CAAC;YACpB,KAAK,EAAE,OAAC,CAAC,KAAK,CAAC,OAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE;YACrE,QAAQ,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,+CAA+C,CAAC;YACzG,QAAQ,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,QAAQ,CAAC,6BAA6B,CAAC;YAC7E,MAAM,EAAE,OAAC,CAAC,KAAK,CAAC,OAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,QAAQ,CAAC,mCAAmC,CAAC;YACvG,UAAU,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC,uCAAuC,CAAC;YACxF,OAAO,EAAE,OAAC;iBACP,MAAM,CAAC;gBACN,QAAQ,EAAE,OAAC,CAAC,KAAK,CAAC,OAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,QAAQ,CAAC,4EAA4E,CAAC;gBACnJ,MAAM,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,IAAK,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,wDAAwD,CAAC;gBAC1H,UAAU,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,mCAAmC,CAAC;aACrG,CAAC;iBACD,QAAQ,EAAE;YACb,IAAI,EAAE,OAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE;YAC5B,OAAO,EAAE,OAAC,CAAC,MAAM,CAAC,OAAC,CAAC,MAAM,EAAE,EAAE,OAAC,CAAC,KAAK,CAAC,OAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,oCAAoC,CAAC;YAC7H,WAAW,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;YACtD,MAAM,EAAE,OAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE;SAC/B,CAAC;QACF,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,EAAE,UAAU,EAAE,EAAE,EAAE,CACvC,IAAA,gCAAgB,EAAC,KAAK,EAAE;YACtB,YAAY;YACZ,KAAK,EAAE,IAAA,oCAAoB,EAAC,WAAW,CAAC,QAAQ,EAAE,WAAW,CAAC,EAAE,EAAE,WAAW,CAAC,WAAW,CAAC;YAC1F,OAAO,EAAE,WAAW,CAAC,EAAE;YACvB,MAAM,EAAE,WAAW;YACnB,WAAW,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,WAAW,CAAC,KAAK,EAAE,WAAW,CAAC,EAAE,CAAC;YAC1D,UAAU,EAAE,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAC1B,OAAO,CAAC,cAAc,CAAC,CAAC,EAAE,GAAG,EAAE,SAAS,EAAE,KAAK,EAAE,kBAAkB,EAAE,MAAM,EAAE,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,MAAM,EAAE,MAAM,EAAE,GAAG,IAAI,IAAI,KAAK,EAAE,EAAE,CAAC,EAAE,UAAU,CAAC;SAC5J,CAAC;KACL,CAAC,CAAA;AACJ,CAAC"}

package/dist/research/extraction-tool.test.js

@@ -85,6 +85,8 @@ for (const cancelAfterFirst of [false, true]) {
         };
         const result = await (0, extraction_tool_1.createExtractionTool)(context, { id: 'extraction-model', provider: {}, attribution: attribution_1.OPENROUTER_APPS.DEEP_RESEARCH }).execute({
             paths: ['missing.txt', 'first.txt', 'second.txt'], question: 'Is a valve required?', fields: ['found'], outputPath: 'results/rows.jsonl',
+            // One document at a time, so the row order and the stop after cancellation are exact.
+            concurrency: 1,
         }, { toolCallId: 'extract', messages: [] });
         const expectedRows = cancelAfterFirst ? 1 : 2;
         strict_1.default.equal(result.rows, expectedRows);

package/dist/research/extraction-tool.test.js.map

@@ -1 +1 @@
-{"version":3,"file":"extraction-tool.test.js","sourceRoot":"","sources":["../../src/research/extraction-tool.test.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,gEAAuC;AACvC,qCAAwC;AACxC,sDAAwB;AACxB,0DAA4B;AAC5B,yCAA8B;AAC9B,kCAA6C;AAC7C,uEAAwD;AACxD,uDAAuD;AAGvD,uDAAwD;AAExD,KAAK,MAAM,gBAAgB,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,EAAE,CAAC;IAC7C,IAAA,cAAE,EAAC,mBAAmB,gBAAgB,CAAC,CAAC,CAAC,0BAA0B,CAAC,CAAC,CAAC,gDAAgD,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,EAAE;QACpI,MAAM,GAAG,GAAG,MAAM,kBAAE,CAAC,OAAO,CAAC,mBAAI,CAAC,IAAI,CAAC,iBAAE,CAAC,MAAM,EAAE,EAAE,mBAAmB,CAAC,CAAC,CAAA;QACzE,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,kBAAE,CAAC,EAAE,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,CAAA;QAC3D,MAAM,kBAAE,CAAC,SAAS,CAAC,mBAAI,CAAC,IAAI,CAAC,GAAG,EAAE,WAAW,CAAC,EAAE,4BAA4B,CAAC,CAAA;QAC7E,MAAM,kBAAE,CAAC,SAAS,CAAC,mBAAI,CAAC,IAAI,CAAC,GAAG,EAAE,YAAY,CAAC,EAAE,iBAAiB,CAAC,CAAA;QACnE,MAAM,KAAK,GAAG,IAAI,0BAAmB,CAAC;YACpC,UAAU,EAAE,KAAK,IAAI,EAAE,CAAC,CAAC;gBACvB,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,qDAAqD,EAAE,CAAC;gBACxF,YAAY,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE;gBAC9C,KAAK,EAAE;oBACL,WAAW,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,OAAO,EAAE,EAAE,EAAE,SAAS,EAAE,CAAC,EAAE,UAAU,EAAE,CAAC,EAAE;oBACpE,YAAY,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE;iBAClD;gBACD,QAAQ,EAAE,EAAE;aACb,CAAC;SACH,CAAC,CAAA;QACF,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,aAAa,EAAE,sBAAsB,EAAE,CAAC,SAAS,EAAE,OAAO,EAAE,GAAG,EAAE,EAAE;YAC/E,gBAAM,CAAC,KAAK,CAAC,OAAO,EAAE,kBAAkB,CAAC,CAAA;YACzC,gBAAM,CAAC,KAAK,CAAC,GAAG,EAAE,6BAAe,CAAC,aAAa,CAAC,CAAA;YAChD,OAAO,KAAK,CAAA;QACd,CAAC,CAAC,CAAA;QACF,MAAM,IAAI,GAAG,EAAE,SAAS,EAAE,cAAc,EAAE,KAAK,EAAE,IAAI,eAAe,EAAE,EAAE,SAAS,EAAE,KAAK,EAAE,CAAA;QAC1F,MAAM,MAAM,GAAoB,EAAE,CAAA;QAClC,IAAI,UAAU,GAAG,CAAC,CAAA;QAClB,MAAM,OAAO,GAAuB;YAClC,YAAY,EAAE,GAAG;YACjB,WAAW,EAAE,IAAI,CAAC,KAAK,CAAC,MAAM;YAC9B,cAAc,EAAE,CAAC,KAAK,EAAE,UAAU,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,UAAU,EAAE,SAAS,EAAE,IAAI,CAAC,SAAS,EAAE,UAAU,EAAE,KAAK,EAAE,CAAC;YACtH,oBAAoB,KAAI,CAAC;YACzB,YAAY,KAAI,CAAC;YACjB,WAAW,CAAC,KAAK,EAAE,OAAO;gBACxB,UAAU,IAAI,KAAK,CAAC,WAAW,CAAA;gBAC/B,gBAAM,CAAC,KAAK,CAAC,OAAO,EAAE,kBAAkB,CAAC,CAAA;gBACzC,IAAI,gBAAgB;oBAAE,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAA;YAC1C,CAAC;SACF,CAAA;QACD,MAAM,MAAM,GAAG,MAAM,IAAA,sCAAoB,EAAC,OAAO,EAAE,EAAE,EAAE,EAAE,kBAAkB,EAAE,QAAQ,EAAE,EAAE,EAAE,WAAW,EAAE,6BAAe,CAAC,aAAa,EAAE,CAAC,CAAC,OAAQ,CAAC;YAChJ,KAAK,EAAE,CAAC,aAAa,EAAE,WAAW,EAAE,YAAY,CAAC,EAAE,QAAQ,EAAE,sBAAsB,EAAE,MAAM,EAAE,CAAC,OAAO,CAAC,EAAE,UAAU,EAAE,oBAAoB;SACzI,EAAE,EAAE,UAAU,EAAE,SAAS,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAwD,CAAA;QAClG,MAAM,YAAY,GAAG,gBAAgB,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAA;QAC7C,gBAAM,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,EAAE,YAAY,CAAC,CAAA;QACvC,gBAAM,CAAC,SAAS,CAAC,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC,aAAa,CAAC,CAAC,CAAA;QACjF,gBAAM,CAAC,KAAK,CAAC,KAAK,CAAC,eAAe,CAAC,MAAM,EAAE,YAAY,CAAC,CAAA;QACxD,gBAAM,CAAC,KAAK,CAAC,UAAU,EAAE,EAAE,GAAG,YAAY,CAAC,CAAA;QAC3C,MAAM,IAAI,GAAG,CAAC,MAAM,kBAAE,CAAC,QAAQ,CAAC,mBAAI,CAAC,IAAI,CAAC,GAAG,EAAE,oBAAoB,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAA;QACjI,gBAAM,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,YAAY,CAAC,CAAA;QACvC,gBAAM,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,MAAM,EAAE,WAAW,EAAE,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,4BAA4B,EAAE,CAAC,CAAA;QACpG,gBAAM,CAAC,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,KAAK,UAAU,IAAI,KAAK,CAAC,SAAS,KAAK,cAAc,IAAI,KAAK,CAAC,UAAU,KAAK,SAAS,CAAC,CAAC,CAAA;IACvI,CAAC,CAAC,CAAA;AACJ,CAAC"}
+{"version":3,"file":"extraction-tool.test.js","sourceRoot":"","sources":["../../src/research/extraction-tool.test.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,gEAAuC;AACvC,qCAAwC;AACxC,sDAAwB;AACxB,0DAA4B;AAC5B,yCAA8B;AAC9B,kCAA6C;AAC7C,uEAAwD;AACxD,uDAAuD;AAGvD,uDAAwD;AAExD,KAAK,MAAM,gBAAgB,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,EAAE,CAAC;IAC7C,IAAA,cAAE,EAAC,mBAAmB,gBAAgB,CAAC,CAAC,CAAC,0BAA0B,CAAC,CAAC,CAAC,gDAAgD,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,EAAE;QACpI,MAAM,GAAG,GAAG,MAAM,kBAAE,CAAC,OAAO,CAAC,mBAAI,CAAC,IAAI,CAAC,iBAAE,CAAC,MAAM,EAAE,EAAE,mBAAmB,CAAC,CAAC,CAAA;QACzE,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,kBAAE,CAAC,EAAE,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,CAAA;QAC3D,MAAM,kBAAE,CAAC,SAAS,CAAC,mBAAI,CAAC,IAAI,CAAC,GAAG,EAAE,WAAW,CAAC,EAAE,4BAA4B,CAAC,CAAA;QAC7E,MAAM,kBAAE,CAAC,SAAS,CAAC,mBAAI,CAAC,IAAI,CAAC,GAAG,EAAE,YAAY,CAAC,EAAE,iBAAiB,CAAC,CAAA;QACnE,MAAM,KAAK,GAAG,IAAI,0BAAmB,CAAC;YACpC,UAAU,EAAE,KAAK,IAAI,EAAE,CAAC,CAAC;gBACvB,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,qDAAqD,EAAE,CAAC;gBACxF,YAAY,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE;gBAC9C,KAAK,EAAE;oBACL,WAAW,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,OAAO,EAAE,EAAE,EAAE,SAAS,EAAE,CAAC,EAAE,UAAU,EAAE,CAAC,EAAE;oBACpE,YAAY,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE;iBAClD;gBACD,QAAQ,EAAE,EAAE;aACb,CAAC;SACH,CAAC,CAAA;QACF,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,aAAa,EAAE,sBAAsB,EAAE,CAAC,SAAS,EAAE,OAAO,EAAE,GAAG,EAAE,EAAE;YAC/E,gBAAM,CAAC,KAAK,CAAC,OAAO,EAAE,kBAAkB,CAAC,CAAA;YACzC,gBAAM,CAAC,KAAK,CAAC,GAAG,EAAE,6BAAe,CAAC,aAAa,CAAC,CAAA;YAChD,OAAO,KAAK,CAAA;QACd,CAAC,CAAC,CAAA;QACF,MAAM,IAAI,GAAG,EAAE,SAAS,EAAE,cAAc,EAAE,KAAK,EAAE,IAAI,eAAe,EAAE,EAAE,SAAS,EAAE,KAAK,EAAE,CAAA;QAC1F,MAAM,MAAM,GAAoB,EAAE,CAAA;QAClC,IAAI,UAAU,GAAG,CAAC,CAAA;QAClB,MAAM,OAAO,GAAuB;YAClC,YAAY,EAAE,GAAG;YACjB,WAAW,EAAE,IAAI,CAAC,KAAK,CAAC,MAAM;YAC9B,cAAc,EAAE,CAAC,KAAK,EAAE,UAAU,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,UAAU,EAAE,SAAS,EAAE,IAAI,CAAC,SAAS,EAAE,UAAU,EAAE,KAAK,EAAE,CAAC;YACtH,oBAAoB,KAAI,CAAC;YACzB,YAAY,KAAI,CAAC;YACjB,WAAW,CAAC,KAAK,EAAE,OAAO;gBACxB,UAAU,IAAI,KAAK,CAAC,WAAW,CAAA;gBAC/B,gBAAM,CAAC,KAAK,CAAC,OAAO,EAAE,kBAAkB,CAAC,CAAA;gBACzC,IAAI,gBAAgB;oBAAE,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAA;YAC1C,CAAC;SACF,CAAA;QACD,MAAM,MAAM,GAAG,MAAM,IAAA,sCAAoB,EAAC,OAAO,EAAE,EAAE,EAAE,EAAE,kBAAkB,EAAE,QAAQ,EAAE,EAAE,EAAE,WAAW,EAAE,6BAAe,CAAC,aAAa,EAAE,CAAC,CAAC,OAAQ,CAAC;YAChJ,KAAK,EAAE,CAAC,aAAa,EAAE,WAAW,EAAE,YAAY,CAAC,EAAE,QAAQ,EAAE,sBAAsB,EAAE,MAAM,EAAE,CAAC,OAAO,CAAC,EAAE,UAAU,EAAE,oBAAoB;YACxI,sFAAsF;YACtF,WAAW,EAAE,CAAC;SACf,EAAE,EAAE,UAAU,EAAE,SAAS,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAwD,CAAA;QAClG,MAAM,YAAY,GAAG,gBAAgB,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAA;QAC7C,gBAAM,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,EAAE,YAAY,CAAC,CAAA;QACvC,gBAAM,CAAC,SAAS,CAAC,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC,aAAa,CAAC,CAAC,CAAA;QACjF,gBAAM,CAAC,KAAK,CAAC,KAAK,CAAC,eAAe,CAAC,MAAM,EAAE,YAAY,CAAC,CAAA;QACxD,gBAAM,CAAC,KAAK,CAAC,UAAU,EAAE,EAAE,GAAG,YAAY,CAAC,CAAA;QAC3C,MAAM,IAAI,GAAG,CAAC,MAAM,kBAAE,CAAC,QAAQ,CAAC,mBAAI,CAAC,IAAI,CAAC,GAAG,EAAE,oBAAoB,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAA;QACjI,gBAAM,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,YAAY,CAAC,CAAA;QACvC,gBAAM,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,MAAM,EAAE,WAAW,EAAE,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,4BAA4B,EAAE,CAAC,CAAA;QACpG,gBAAM,CAAC,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,KAAK,UAAU,IAAI,KAAK,CAAC,SAAS,KAAK,cAAc,IAAI,KAAK,CAAC,UAAU,KAAK,SAAS,CAAC,CAAC,CAAA;IACvI,CAAC,CAAC,CAAA;AACJ,CAAC"}

package/dist/research/filingramp/attachment-tools.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"attachment-tools.d.ts","sourceRoot":"","sources":["../../../src/research/filingramp/attachment-tools.ts"],"names":[],"mappings":"AAIA,OAAO,EAAQ,KAAK,IAAI,EAAE,MAAM,IAAI,CAAA;AAEpC,OAAO,EAAgC,KAAK,sBAAsB,EAAE,MAAM,UAAU,CAAA;AAuBpF,wBAAgB,qBAAqB,CAAC,OAAO,EAAE,sBAAsB,GAAG,MAAM,CAAC,MAAM,EAAE,IAAI,CAAC,CAgE3F"}
+{"version":3,"file":"attachment-tools.d.ts","sourceRoot":"","sources":["../../../src/research/filingramp/attachment-tools.ts"],"names":[],"mappings":"AAIA,OAAO,EAAQ,KAAK,IAAI,EAAE,MAAM,IAAI,CAAA;AAGpC,OAAO,EAA+D,KAAK,sBAAsB,EAAE,MAAM,UAAU,CAAA;AAyBnH,wBAAgB,qBAAqB,CAAC,OAAO,EAAE,sBAAsB,GAAG,MAAM,CAAC,MAAM,EAAE,IAAI,CAAC,CAyJ3F"}