@staticn0va/wigolo 0.6.6 → 1.0.1

package/dist/extraction/llm/gemini.js

@@ -0,0 +1,35 @@
+import { GoogleGenAI } from '@google/genai';
+const DEFAULT_MODEL = 'gemini-2.5-flash-lite';
+export async function callGemini(opts, apiKey) {
+    const client = new GoogleGenAI({ apiKey });
+    const model = opts.modelOverride ?? DEFAULT_MODEL;
+    const start = Date.now();
+    const response = await client.models.generateContent({
+        model,
+        contents: opts.prompt,
+        config: {
+            responseMimeType: 'application/json',
+            responseJsonSchema: opts.jsonSchema,
+            abortSignal: opts.signal,
+        },
+    });
+    const text = response.text;
+    if (!text) {
+        throw new Error('gemini: empty text in response');
+    }
+    let values;
+    try {
+        values = JSON.parse(text);
+    }
+    catch (e) {
+        throw new Error(`gemini: invalid JSON in response: ${e.message}`);
+    }
+    return {
+        values,
+        provider: 'gemini',
+        model,
+        cached: false,
+        latencyMs: Date.now() - start,
+    };
+}
+//# sourceMappingURL=gemini.js.map

package/dist/extraction/llm/gemini.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"gemini.js","sourceRoot":"","sources":["../../../src/extraction/llm/gemini.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,eAAe,CAAC;AAG5C,MAAM,aAAa,GAAG,uBAAuB,CAAC;AAE9C,MAAM,CAAC,KAAK,UAAU,UAAU,CAC9B,IAAiB,EACjB,MAAc;IAEd,MAAM,MAAM,GAAG,IAAI,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC;IAC3C,MAAM,KAAK,GAAG,IAAI,CAAC,aAAa,IAAI,aAAa,CAAC;IAClD,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAEzB,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,MAAM,CAAC,eAAe,CAAC;QACnD,KAAK;QACL,QAAQ,EAAE,IAAI,CAAC,MAAM;QACrB,MAAM,EAAE;YACN,gBAAgB,EAAE,kBAAkB;YACpC,kBAAkB,EAAE,IAAI,CAAC,UAAU;YACnC,WAAW,EAAE,IAAI,CAAC,MAAM;SACzB;KACF,CAAC,CAAC;IAEH,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI,CAAC;IAC3B,IAAI,CAAC,IAAI,EAAE,CAAC;QACV,MAAM,IAAI,KAAK,CAAC,gCAAgC,CAAC,CAAC;IACpD,CAAC;IAED,IAAI,MAA+B,CAAC;IACpC,IAAI,CAAC;QACH,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAC5B,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,MAAM,IAAI,KAAK,CAAC,qCAAsC,CAAW,CAAC,OAAO,EAAE,CAAC,CAAC;IAC/E,CAAC;IAED,OAAO;QACL,MAAM;QACN,QAAQ,EAAE,QAAQ;QAClB,KAAK;QACL,MAAM,EAAE,KAAK;QACb,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK;KAC9B,CAAC;AACJ,CAAC"}

package/dist/extraction/llm/groq.d.ts

@@ -0,0 +1,3 @@
+import type { LLMCallOpts, LLMExtractResult } from './types.js';
+export declare function callGroq(opts: LLMCallOpts, apiKey: string): Promise<LLMExtractResult>;
+//# sourceMappingURL=groq.d.ts.map

package/dist/extraction/llm/groq.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"groq.d.ts","sourceRoot":"","sources":["../../../src/extraction/llm/groq.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,WAAW,EAAE,gBAAgB,EAAE,MAAM,YAAY,CAAC;AAKhE,wBAAsB,QAAQ,CAC5B,IAAI,EAAE,WAAW,EACjB,MAAM,EAAE,MAAM,GACb,OAAO,CAAC,gBAAgB,CAAC,CA2B3B"}

package/dist/extraction/llm/groq.js

@@ -0,0 +1,63 @@
+import Groq from 'groq-sdk';
+import { validateAgainstSchema } from './validate.js';
+const DEFAULT_MODEL = 'llama-3.3-70b-versatile';
+export async function callGroq(opts, apiKey) {
+    const client = new Groq({ apiKey });
+    const model = opts.modelOverride ?? DEFAULT_MODEL;
+    const start = Date.now();
+    const messages = [
+        { role: 'user', content: buildPrompt(opts.prompt, opts.jsonSchema) },
+    ];
+    const first = await runOnce(client, model, messages, opts.signal);
+    let errors = validateAgainstSchema(first.values, opts.jsonSchema);
+    if (errors.length === 0) {
+        return done(first.values, first.responseModel ?? model, start);
+    }
+    // Retry once with validation errors fed back to the model.
+    messages.push({ role: 'assistant', content: first.raw });
+    messages.push({ role: 'user', content: retryPrompt(errors) });
+    const second = await runOnce(client, model, messages, opts.signal);
+    errors = validateAgainstSchema(second.values, opts.jsonSchema);
+    if (errors.length > 0) {
+        throw new Error(`groq: response failed schema validation after retry: ${formatErrors(errors)}`);
+    }
+    return done(second.values, second.responseModel ?? model, start);
+}
+async function runOnce(client, model, messages, signal) {
+    const response = await client.chat.completions.create({
+        model,
+        messages,
+        response_format: { type: 'json_object' },
+    }, { signal });
+    const content = response.choices?.[0]?.message?.content;
+    if (!content) {
+        throw new Error('groq: empty content in response');
+    }
+    let values;
+    try {
+        values = JSON.parse(content);
+    }
+    catch (e) {
+        throw new Error(`groq: invalid JSON in response: ${e.message}`);
+    }
+    return { values, raw: content, responseModel: response.model };
+}
+function buildPrompt(prompt, schema) {
+    return `${prompt}\n\nReturn JSON matching this schema:\n${JSON.stringify(schema)}`;
+}
+function retryPrompt(errors) {
+    return `Your previous response failed schema validation:\n${formatErrors(errors)}\nReturn corrected JSON only.`;
+}
+function formatErrors(errors) {
+    return errors.map((e) => `${e.path}: ${e.message}`).join('; ');
+}
+function done(values, model, start) {
+    return {
+        values,
+        provider: 'groq',
+        model,
+        cached: false,
+        latencyMs: Date.now() - start,
+    };
+}
+//# sourceMappingURL=groq.js.map

package/dist/extraction/llm/groq.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"groq.js","sourceRoot":"","sources":["../../../src/extraction/llm/groq.ts"],"names":[],"mappings":"AAAA,OAAO,IAAI,MAAM,UAAU,CAAC;AAE5B,OAAO,EAAE,qBAAqB,EAAwB,MAAM,eAAe,CAAC;AAE5E,MAAM,aAAa,GAAG,yBAAyB,CAAC;AAEhD,MAAM,CAAC,KAAK,UAAU,QAAQ,CAC5B,IAAiB,EACjB,MAAc;IAEd,MAAM,MAAM,GAAG,IAAI,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC;IACpC,MAAM,KAAK,GAAG,IAAI,CAAC,aAAa,IAAI,aAAa,CAAC;IAClD,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAEzB,MAAM,QAAQ,GAA2D;QACvE,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,WAAW,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,UAAU,CAAC,EAAE;KACrE,CAAC;IAEF,MAAM,KAAK,GAAG,MAAM,OAAO,CAAC,MAAM,EAAE,KAAK,EAAE,QAAQ,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;IAClE,IAAI,MAAM,GAAG,qBAAqB,CAAC,KAAK,CAAC,MAAM,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC;IAClE,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACxB,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,aAAa,IAAI,KAAK,EAAE,KAAK,CAAC,CAAC;IACjE,CAAC;IAED,2DAA2D;IAC3D,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,WAAW,EAAE,OAAO,EAAE,KAAK,CAAC,GAAG,EAAE,CAAC,CAAC;IACzD,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,WAAW,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;IAE9D,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,MAAM,EAAE,KAAK,EAAE,QAAQ,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;IACnE,MAAM,GAAG,qBAAqB,CAAC,MAAM,CAAC,MAAM,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC;IAC/D,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACtB,MAAM,IAAI,KAAK,CACb,wDAAwD,YAAY,CAAC,MAAM,CAAC,EAAE,CAC/E,CAAC;IACJ,CAAC;IACD,OAAO,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,aAAa,IAAI,KAAK,EAAE,KAAK,CAAC,CAAC;AACnE,CAAC;AAQD,KAAK,UAAU,OAAO,CACpB,MAAY,EACZ,KAAa,EACb,QAAgE,EAChE,MAA+B;IAE/B,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,MAAM,CACnD;QACE,KAAK;QACL,QAAQ;QACR,eAAe,EAAE,EAAE,IAAI,EAAE,aAAa,EAAE;KACzC,EACD,EAAE,MAAM,EAAE,CACX,CAAC;IACF,MAAM,OAAO,GAAG,QAAQ,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,OAAO,CAAC;IACxD,IAAI,CAAC,OAAO,EAAE,CAAC;QACb,MAAM,IAAI,KAAK,CAAC,iCAAiC,CAAC,CAAC;IACrD,CAAC;IACD,IAAI,MAA+B,CAAC;IACpC,IAAI,CAAC;QACH,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;IAC/B,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,MAAM,IAAI,KAAK,CAAC,mCAAoC,CAAW,CAAC,OAAO,EAAE,CAAC,CAAC;IAC7E,CAAC;IACD,OAAO,EAAE,MAAM,EAAE,GAAG,EAAE,OAAO,EAAE,aAAa,EAAE,QAAQ,CAAC,KAAK,EAAE,CAAC;AACjE,CAAC;AAED,SAAS,WAAW,CAAC,MAAc,EAAE,MAA+B;IAClE,OAAO,GAAG,MAAM,0CAA0C,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC;AACrF,CAAC;AAED,SAAS,WAAW,CAAC,MAAyB;IAC5C,OAAO,qDAAqD,YAAY,CAAC,MAAM,CAAC,+BAA+B,CAAC;AAClH,CAAC;AAED,SAAS,YAAY,CAAC,MAAyB;IAC7C,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AACjE,CAAC;AAED,SAAS,IAAI,CACX,MAA+B,EAC/B,KAAa,EACb,KAAa;IAEb,OAAO;QACL,MAAM;QACN,QAAQ,EAAE,MAAM;QAChB,KAAK;QACL,MAAM,EAAE,KAAK;QACb,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK;KAC9B,CAAC;AACJ,CAAC"}

package/dist/extraction/llm/hash.d.ts

@@ -0,0 +1,3 @@
+export declare function hashPrompt(prompt: string): string;
+export declare function hashSchema(schema: unknown): string;
+//# sourceMappingURL=hash.d.ts.map

package/dist/extraction/llm/hash.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"hash.d.ts","sourceRoot":"","sources":["../../../src/extraction/llm/hash.ts"],"names":[],"mappings":"AAEA,wBAAgB,UAAU,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAGjD;AAED,wBAAgB,UAAU,CAAC,MAAM,EAAE,OAAO,GAAG,MAAM,CAElD"}

package/dist/extraction/llm/hash.js

@@ -0,0 +1,22 @@
+import { createHash } from 'node:crypto';
+export function hashPrompt(prompt) {
+    const normalized = prompt.replace(/\s+/g, ' ').trim();
+    return createHash('sha256').update(normalized).digest('hex');
+}
+export function hashSchema(schema) {
+    return createHash('sha256').update(stableStringify(schema)).digest('hex');
+}
+function stableStringify(value) {
+    if (value === null || typeof value !== 'object') {
+        return JSON.stringify(value);
+    }
+    if (Array.isArray(value)) {
+        return '[' + value.map(stableStringify).join(',') + ']';
+    }
+    const keys = Object.keys(value).sort();
+    const parts = keys.map((k) => JSON.stringify(k) +
+        ':' +
+        stableStringify(value[k]));
+    return '{' + parts.join(',') + '}';
+}
+//# sourceMappingURL=hash.js.map

package/dist/extraction/llm/hash.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"hash.js","sourceRoot":"","sources":["../../../src/extraction/llm/hash.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAEzC,MAAM,UAAU,UAAU,CAAC,MAAc;IACvC,MAAM,UAAU,GAAG,MAAM,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;IACtD,OAAO,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;AAC/D,CAAC;AAED,MAAM,UAAU,UAAU,CAAC,MAAe;IACxC,OAAO,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;AAC5E,CAAC;AAED,SAAS,eAAe,CAAC,KAAc;IACrC,IAAI,KAAK,KAAK,IAAI,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;QAChD,OAAO,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;IAC/B,CAAC;IACD,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;QACzB,OAAO,GAAG,GAAG,KAAK,CAAC,GAAG,CAAC,eAAe,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,GAAG,CAAC;IAC1D,CAAC;IACD,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,KAAgC,CAAC,CAAC,IAAI,EAAE,CAAC;IAClE,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CACpB,CAAC,CAAC,EAAE,EAAE,CACJ,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC;QACjB,GAAG;QACH,eAAe,CAAE,KAAiC,CAAC,CAAC,CAAC,CAAC,CACzD,CAAC;IACF,OAAO,GAAG,GAAG,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,GAAG,CAAC;AACrC,CAAC"}

package/dist/extraction/llm/openai.d.ts

@@ -0,0 +1,3 @@
+import type { LLMCallOpts, LLMExtractResult } from './types.js';
+export declare function callOpenAI(opts: LLMCallOpts, apiKey: string): Promise<LLMExtractResult>;
+//# sourceMappingURL=openai.d.ts.map

package/dist/extraction/llm/openai.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"openai.d.ts","sourceRoot":"","sources":["../../../src/extraction/llm/openai.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,WAAW,EAAE,gBAAgB,EAAE,MAAM,YAAY,CAAC;AAIhE,wBAAsB,UAAU,CAC9B,IAAI,EAAE,WAAW,EACjB,MAAM,EAAE,MAAM,GACb,OAAO,CAAC,gBAAgB,CAAC,CAwC3B"}

package/dist/extraction/llm/openai.js

@@ -0,0 +1,38 @@
+import OpenAI from 'openai';
+const DEFAULT_MODEL = 'gpt-4o-mini';
+export async function callOpenAI(opts, apiKey) {
+    const client = new OpenAI({ apiKey });
+    const model = opts.modelOverride ?? DEFAULT_MODEL;
+    const start = Date.now();
+    const response = await client.chat.completions.create({
+        model,
+        messages: [{ role: 'user', content: opts.prompt }],
+        response_format: {
+            type: 'json_schema',
+            json_schema: {
+                name: 'extract',
+                schema: opts.jsonSchema,
+                strict: true,
+            },
+        },
+    }, { signal: opts.signal });
+    const content = response.choices?.[0]?.message?.content;
+    if (!content) {
+        throw new Error('openai: empty content in response');
+    }
+    let values;
+    try {
+        values = JSON.parse(content);
+    }
+    catch (e) {
+        throw new Error(`openai: invalid JSON in response: ${e.message}`);
+    }
+    return {
+        values,
+        provider: 'openai',
+        model: response.model ?? model,
+        cached: false,
+        latencyMs: Date.now() - start,
+    };
+}
+//# sourceMappingURL=openai.js.map

package/dist/extraction/llm/openai.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"openai.js","sourceRoot":"","sources":["../../../src/extraction/llm/openai.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,MAAM,QAAQ,CAAC;AAG5B,MAAM,aAAa,GAAG,aAAa,CAAC;AAEpC,MAAM,CAAC,KAAK,UAAU,UAAU,CAC9B,IAAiB,EACjB,MAAc;IAEd,MAAM,MAAM,GAAG,IAAI,MAAM,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC;IACtC,MAAM,KAAK,GAAG,IAAI,CAAC,aAAa,IAAI,aAAa,CAAC;IAClD,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAEzB,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,MAAM,CACnD;QACE,KAAK;QACL,QAAQ,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC;QAClD,eAAe,EAAE;YACf,IAAI,EAAE,aAAa;YACnB,WAAW,EAAE;gBACX,IAAI,EAAE,SAAS;gBACf,MAAM,EAAE,IAAI,CAAC,UAAU;gBACvB,MAAM,EAAE,IAAI;aACb;SACF;KACF,EACD,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,CACxB,CAAC;IAEF,MAAM,OAAO,GAAG,QAAQ,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,OAAO,CAAC;IACxD,IAAI,CAAC,OAAO,EAAE,CAAC;QACb,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;IACvD,CAAC;IAED,IAAI,MAA+B,CAAC;IACpC,IAAI,CAAC;QACH,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;IAC/B,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,MAAM,IAAI,KAAK,CAAC,qCAAsC,CAAW,CAAC,OAAO,EAAE,CAAC,CAAC;IAC/E,CAAC;IAED,OAAO;QACL,MAAM;QACN,QAAQ,EAAE,QAAQ;QAClB,KAAK,EAAE,QAAQ,CAAC,KAAK,IAAI,KAAK;QAC9B,MAAM,EAAE,KAAK;QACb,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK;KAC9B,CAAC;AACJ,CAAC"}

package/dist/extraction/llm/select.d.ts

@@ -0,0 +1,5 @@
+import type { LLMProvider } from './types.js';
+export declare function selectProvider(env: Record<string, string | undefined>): LLMProvider | null;
+export declare function providerEnvVar(p: LLMProvider): string;
+export declare function allProviders(): readonly LLMProvider[];
+//# sourceMappingURL=select.d.ts.map

package/dist/extraction/llm/select.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"select.d.ts","sourceRoot":"","sources":["../../../src/extraction/llm/select.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AAW9C,wBAAgB,cAAc,CAC5B,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC,GACtC,WAAW,GAAG,IAAI,CAUpB;AAED,wBAAgB,cAAc,CAAC,CAAC,EAAE,WAAW,GAAG,MAAM,CAErD;AAED,wBAAgB,YAAY,IAAI,SAAS,WAAW,EAAE,CAErD"}

package/dist/extraction/llm/select.js

@@ -0,0 +1,27 @@
+const PROVIDER_ORDER = ['anthropic', 'openai', 'gemini', 'groq'];
+const PROVIDER_ENV = {
+    anthropic: 'ANTHROPIC_API_KEY',
+    openai: 'OPENAI_API_KEY',
+    gemini: 'GOOGLE_API_KEY',
+    groq: 'GROQ_API_KEY',
+};
+export function selectProvider(env) {
+    const override = env.WIGOLO_LLM_PROVIDER;
+    if (override && PROVIDER_ORDER.includes(override)) {
+        const p = override;
+        if (env[PROVIDER_ENV[p]])
+            return p;
+    }
+    for (const p of PROVIDER_ORDER) {
+        if (env[PROVIDER_ENV[p]])
+            return p;
+    }
+    return null;
+}
+export function providerEnvVar(p) {
+    return PROVIDER_ENV[p];
+}
+export function allProviders() {
+    return PROVIDER_ORDER;
+}
+//# sourceMappingURL=select.js.map

package/dist/extraction/llm/select.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"select.js","sourceRoot":"","sources":["../../../src/extraction/llm/select.ts"],"names":[],"mappings":"AAEA,MAAM,cAAc,GAAkB,CAAC,WAAW,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,CAAC,CAAC;AAEhF,MAAM,YAAY,GAAgC;IAChD,SAAS,EAAE,mBAAmB;IAC9B,MAAM,EAAE,gBAAgB;IACxB,MAAM,EAAE,gBAAgB;IACxB,IAAI,EAAE,cAAc;CACrB,CAAC;AAEF,MAAM,UAAU,cAAc,CAC5B,GAAuC;IAEvC,MAAM,QAAQ,GAAG,GAAG,CAAC,mBAAmB,CAAC;IACzC,IAAI,QAAQ,IAAK,cAA2B,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC;QAChE,MAAM,CAAC,GAAG,QAAuB,CAAC;QAClC,IAAI,GAAG,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;YAAE,OAAO,CAAC,CAAC;IACrC,CAAC;IACD,KAAK,MAAM,CAAC,IAAI,cAAc,EAAE,CAAC;QAC/B,IAAI,GAAG,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;YAAE,OAAO,CAAC,CAAC;IACrC,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,MAAM,UAAU,cAAc,CAAC,CAAc;IAC3C,OAAO,YAAY,CAAC,CAAC,CAAC,CAAC;AACzB,CAAC;AAED,MAAM,UAAU,YAAY;IAC1B,OAAO,cAAc,CAAC;AACxB,CAAC"}

package/dist/extraction/llm/types.d.ts

@@ -0,0 +1,24 @@
+export type LLMProvider = 'anthropic' | 'openai' | 'gemini' | 'groq';
+export interface LLMExtractResult {
+    values: Record<string, unknown>;
+    provider: LLMProvider;
+    model: string;
+    cached: boolean;
+    latencyMs: number;
+    warnings?: string[];
+}
+export interface LLMCallRecord {
+    modelId: string;
+    promptHash: string;
+    schemaHash: string;
+    response: string;
+    createdAt: number;
+    expiresAt: number;
+}
+export interface LLMCallOpts {
+    prompt: string;
+    jsonSchema: Record<string, unknown>;
+    modelOverride?: string;
+    signal?: AbortSignal;
+}
+//# sourceMappingURL=types.d.ts.map

package/dist/extraction/llm/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../../src/extraction/llm/types.ts"],"names":[],"mappings":"AAAA,MAAM,MAAM,WAAW,GAAG,WAAW,GAAG,QAAQ,GAAG,QAAQ,GAAG,MAAM,CAAC;AAErE,MAAM,WAAW,gBAAgB;IAC/B,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAChC,QAAQ,EAAE,WAAW,CAAC;IACtB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,OAAO,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,EAAE,CAAC;CACrB;AAED,MAAM,WAAW,aAAa;IAC5B,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,WAAW;IAC1B,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACpC,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,MAAM,CAAC,EAAE,WAAW,CAAC;CACtB"}

package/dist/extraction/llm/types.js

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=types.js.map

package/dist/extraction/llm/types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../../../src/extraction/llm/types.ts"],"names":[],"mappings":""}

package/dist/extraction/llm/validate.d.ts

@@ -0,0 +1,6 @@
+export interface ValidationError {
+    path: string;
+    message: string;
+}
+export declare function validateAgainstSchema(value: unknown, schema: Record<string, unknown>): ValidationError[];
+//# sourceMappingURL=validate.d.ts.map

package/dist/extraction/llm/validate.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"validate.d.ts","sourceRoot":"","sources":["../../../src/extraction/llm/validate.ts"],"names":[],"mappings":"AAIA,MAAM,WAAW,eAAe;IAC9B,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,MAAM,CAAC;CACjB;AASD,wBAAgB,qBAAqB,CACnC,KAAK,EAAE,OAAO,EACd,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAC9B,eAAe,EAAE,CAInB"}

package/dist/extraction/llm/validate.js

@@ -0,0 +1,63 @@
+// Minimal recursive JSON Schema validator: required + type checks.
+// Sufficient for post-hoc validation of provider responses where the SDK
+// does not enforce a schema natively (e.g. Groq json_object).
+export function validateAgainstSchema(value, schema) {
+    const errors = [];
+    walk(value, schema, '$', errors);
+    return errors;
+}
+function walk(value, schema, path, errors) {
+    if (schema.type) {
+        const types = Array.isArray(schema.type) ? schema.type : [schema.type];
+        if (!types.some((t) => matchesType(value, t))) {
+            errors.push({
+                path,
+                message: `expected type ${types.join('|')} but got ${actualType(value)}`,
+            });
+            return;
+        }
+    }
+    if (schema.type === 'object' && value && typeof value === 'object') {
+        const obj = value;
+        for (const req of schema.required ?? []) {
+            if (obj[req] === undefined) {
+                errors.push({ path: `${path}.${req}`, message: 'required' });
+            }
+        }
+        for (const [k, sub] of Object.entries(schema.properties ?? {})) {
+            if (obj[k] !== undefined) {
+                walk(obj[k], sub, `${path}.${k}`, errors);
+            }
+        }
+    }
+    if (schema.type === 'array' && Array.isArray(value) && schema.items) {
+        value.forEach((item, i) => walk(item, schema.items, `${path}[${i}]`, errors));
+    }
+}
+function matchesType(value, type) {
+    switch (type) {
+        case 'string':
+            return typeof value === 'string';
+        case 'number':
+        case 'integer':
+            return typeof value === 'number';
+        case 'boolean':
+            return typeof value === 'boolean';
+        case 'null':
+            return value === null;
+        case 'array':
+            return Array.isArray(value);
+        case 'object':
+            return value !== null && typeof value === 'object' && !Array.isArray(value);
+        default:
+            return true;
+    }
+}
+function actualType(value) {
+    if (value === null)
+        return 'null';
+    if (Array.isArray(value))
+        return 'array';
+    return typeof value;
+}
+//# sourceMappingURL=validate.js.map

package/dist/extraction/llm/validate.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"validate.js","sourceRoot":"","sources":["../../../src/extraction/llm/validate.ts"],"names":[],"mappings":"AAAA,mEAAmE;AACnE,yEAAyE;AACzE,8DAA8D;AAc9D,MAAM,UAAU,qBAAqB,CACnC,KAAc,EACd,MAA+B;IAE/B,MAAM,MAAM,GAAsB,EAAE,CAAC;IACrC,IAAI,CAAC,KAAK,EAAE,MAAqB,EAAE,GAAG,EAAE,MAAM,CAAC,CAAC;IAChD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,IAAI,CACX,KAAc,EACd,MAAmB,EACnB,IAAY,EACZ,MAAyB;IAEzB,IAAI,MAAM,CAAC,IAAI,EAAE,CAAC;QAChB,MAAM,KAAK,GAAG,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QACvE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,WAAW,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAC9C,MAAM,CAAC,IAAI,CAAC;gBACV,IAAI;gBACJ,OAAO,EAAE,iBAAiB,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,YAAY,UAAU,CAAC,KAAK,CAAC,EAAE;aACzE,CAAC,CAAC;YACH,OAAO;QACT,CAAC;IACH,CAAC;IAED,IAAI,MAAM,CAAC,IAAI,KAAK,QAAQ,IAAI,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;QACnE,MAAM,GAAG,GAAG,KAAgC,CAAC;QAC7C,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,QAAQ,IAAI,EAAE,EAAE,CAAC;YACxC,IAAI,GAAG,CAAC,GAAG,CAAC,KAAK,SAAS,EAAE,CAAC;gBAC3B,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,GAAG,IAAI,IAAI,GAAG,EAAE,EAAE,OAAO,EAAE,UAAU,EAAE,CAAC,CAAC;YAC/D,CAAC;QACH,CAAC;QACD,KAAK,MAAM,CAAC,CAAC,EAAE,GAAG,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,UAAU,IAAI,EAAE,CAAC,EAAE,CAAC;YAC/D,IAAI,GAAG,CAAC,CAAC,CAAC,KAAK,SAAS,EAAE,CAAC;gBACzB,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,GAAG,IAAI,IAAI,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC;YAC5C,CAAC;QACH,CAAC;IACH,CAAC;IAED,IAAI,MAAM,CAAC,IAAI,KAAK,OAAO,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,MAAM,CAAC,KAAK,EAAE,CAAC;QACpE,KAAK,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC,EAAE,EAAE,CACxB,IAAI,CAAC,IAAI,EAAE,MAAM,CAAC,KAAoB,EAAE,GAAG,IAAI,IAAI,CAAC,GAAG,EAAE,MAAM,CAAC,CACjE,CAAC;IACJ,CAAC;AACH,CAAC;AAED,SAAS,WAAW,CAAC,KAAc,EAAE,IAAY;IAC/C,QAAQ,IAAI,EAAE,CAAC;QACb,KAAK,QAAQ;YACX,OAAO,OAAO,KAAK,KAAK,QAAQ,CAAC;QACnC,KAAK,QAAQ,CAAC;QACd,KAAK,SAAS;YACZ,OAAO,OAAO,KAAK,KAAK,QAAQ,CAAC;QACnC,KAAK,SAAS;YACZ,OAAO,OAAO,KAAK,KAAK,SAAS,CAAC;QACpC,KAAK,MAAM;YACT,OAAO,KAAK,KAAK,IAAI,CAAC;QACxB,KAAK,OAAO;YACV,OAAO,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;QAC9B,KAAK,QAAQ;YACX,OAAO,KAAK,KAAK,IAAI,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;QAC9E;YACE,OAAO,IAAI,CAAC;IAChB,CAAC;AACH,CAAC;AAED,SAAS,UAAU,CAAC,KAAc;IAChC,IAAI,KAAK,KAAK,IAAI;QAAE,OAAO,MAAM,CAAC;IAClC,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;QAAE,OAAO,OAAO,CAAC;IACzC,OAAO,OAAO,KAAK,CAAC;AACtB,CAAC"}

package/dist/extraction/llm-fallback.d.ts

@@ -0,0 +1,17 @@
+import type { LLMExtractResult } from './llm/types.js';
+export interface LLMFallbackBudget {
+    remaining: number;
+}
+export interface LLMFallbackInput {
+    html: string;
+    jsonSchema: Record<string, unknown>;
+    partial: Record<string, unknown>;
+    missing: string[];
+    signal?: AbortSignal;
+    budget?: LLMFallbackBudget;
+}
+export interface LLMFallbackResult extends LLMExtractResult {
+    warnings: string[];
+}
+export declare function extractWithLLM(input: LLMFallbackInput): Promise<LLMFallbackResult>;
+//# sourceMappingURL=llm-fallback.d.ts.map

package/dist/extraction/llm-fallback.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"llm-fallback.d.ts","sourceRoot":"","sources":["../../src/extraction/llm-fallback.ts"],"names":[],"mappings":"AAYA,OAAO,KAAK,EAAE,gBAAgB,EAAe,MAAM,gBAAgB,CAAC;AAKpE,MAAM,WAAW,iBAAiB;IAChC,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,gBAAgB;IAC/B,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACpC,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACjC,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,MAAM,CAAC,EAAE,iBAAiB,CAAC;CAC5B;AAED,MAAM,WAAW,iBAAkB,SAAQ,gBAAgB;IACzD,QAAQ,EAAE,MAAM,EAAE,CAAC;CACpB;AAeD,wBAAsB,cAAc,CAClC,KAAK,EAAE,gBAAgB,GACtB,OAAO,CAAC,iBAAiB,CAAC,CAsF5B"}

package/dist/extraction/llm-fallback.js

@@ -0,0 +1,129 @@
+import { getConfig } from '../config.js';
+import { callAnthropic } from './llm/anthropic.js';
+import { callOpenAI } from './llm/openai.js';
+import { callGemini } from './llm/gemini.js';
+import { callGroq } from './llm/groq.js';
+import { ensureLLMCacheTable, insertLLMCache, lookupLLMCache, } from './llm/cache.js';
+import { hashPrompt, hashSchema } from './llm/hash.js';
+import { allProviders, providerEnvVar, selectProvider } from './llm/select.js';
+import { validateAgainstSchema } from './llm/validate.js';
+const MAX_HTML_BYTES = 50_000;
+const ADAPTERS = {
+    anthropic: callAnthropic,
+    openai: callOpenAI,
+    gemini: callGemini,
+    groq: callGroq,
+};
+export async function extractWithLLM(input) {
+    if (input.missing.length === 0) {
+        return emptyResult(input.partial, []);
+    }
+    const cfg = getConfig();
+    const budget = input.budget ?? { remaining: cfg.llmMaxCallsPerRequest };
+    if (budget.remaining <= 0) {
+        return emptyResult(input.partial, [
+            `LLM fallback skipped: per-request budget exhausted (cap ${cfg.llmMaxCallsPerRequest}). Override via WIGOLO_LLM_MAX_CALLS_PER_REQUEST.`,
+        ]);
+    }
+    const provider = selectProvider(process.env);
+    if (!provider) {
+        const envList = allProviders()
+            .map((p) => providerEnvVar(p))
+            .join(', ');
+        return emptyResult(input.partial, [
+            `LLM fallback skipped: no provider key set (${envList}). ` +
+                `${input.missing.length} required field(s) still missing: ${input.missing.join(', ')}.`,
+        ]);
+    }
+    const apiKey = process.env[providerEnvVar(provider)];
+    const prompt = buildPrompt(input);
+    const promptHash = hashPrompt(prompt);
+    const schemaHash = hashSchema(input.jsonSchema);
+    const modelId = `${provider}:default`;
+    ensureLLMCacheTable();
+    const cached = lookupLLMCache(modelId, promptHash, schemaHash);
+    if (cached) {
+        const values = JSON.parse(cached);
+        return {
+            values: mergeOnlyMissing(input.partial, values, input.missing),
+            provider,
+            model: modelId,
+            cached: true,
+            latencyMs: 0,
+            warnings: [],
+        };
+    }
+    let result;
+    try {
+        result = await ADAPTERS[provider]({ prompt, jsonSchema: input.jsonSchema, signal: input.signal }, apiKey);
+    }
+    catch (e) {
+        return emptyResult(input.partial, [
+            `LLM fallback (${provider}) failed: ${e.message}`,
+        ]);
+    }
+    finally {
+        budget.remaining = Math.max(0, budget.remaining - 1);
+    }
+    const errors = validateAgainstSchema(result.values, input.jsonSchema);
+    if (errors.length > 0) {
+        return emptyResult(input.partial, [
+            `LLM fallback (${provider}) response failed schema validation: ${errors
+                .map((e) => `${e.path} ${e.message}`)
+                .join('; ')}`,
+        ]);
+    }
+    const ttlMs = cfg.llmCacheTtlDays * 24 * 60 * 60 * 1000;
+    const now = Date.now();
+    insertLLMCache({
+        modelId,
+        promptHash,
+        schemaHash,
+        response: JSON.stringify(result.values),
+        createdAt: now,
+        expiresAt: now + ttlMs,
+    });
+    return {
+        values: mergeOnlyMissing(input.partial, result.values, input.missing),
+        provider,
+        model: result.model,
+        cached: false,
+        latencyMs: result.latencyMs,
+        warnings: result.warnings ?? [],
+    };
+}
+function emptyResult(partial, warnings) {
+    return {
+        values: { ...partial },
+        provider: 'anthropic',
+        model: '',
+        cached: false,
+        latencyMs: 0,
+        warnings,
+    };
+}
+function mergeOnlyMissing(partial, filled, missing) {
+    const out = { ...partial };
+    for (const key of missing) {
+        if (filled[key] !== undefined)
+            out[key] = filled[key];
+    }
+    return out;
+}
+function buildPrompt(input) {
+    const html = truncate(input.html, MAX_HTML_BYTES);
+    return [
+        'Extract the following missing fields from the HTML below.',
+        `Missing fields: ${input.missing.join(', ')}.`,
+        'Return JSON matching the provided schema. Do not invent values; if a field is not present in the HTML, omit it.',
+        '',
+        'HTML:',
+        html,
+    ].join('\n');
+}
+function truncate(s, maxBytes) {
+    if (s.length <= maxBytes)
+        return s;
+    return s.slice(0, maxBytes);
+}
+//# sourceMappingURL=llm-fallback.js.map

package/dist/extraction/llm-fallback.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"llm-fallback.js","sourceRoot":"","sources":["../../src/extraction/llm-fallback.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AACnD,OAAO,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAC7C,OAAO,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAC7C,OAAO,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AACzC,OAAO,EACL,mBAAmB,EACnB,cAAc,EACd,cAAc,GACf,MAAM,gBAAgB,CAAC;AACxB,OAAO,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,eAAe,CAAC;AACvD,OAAO,EAAE,YAAY,EAAE,cAAc,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AAE/E,OAAO,EAAE,qBAAqB,EAAE,MAAM,mBAAmB,CAAC;AAE1D,MAAM,cAAc,GAAG,MAAM,CAAC;AAmB9B,MAAM,QAAQ,GAMV;IACF,SAAS,EAAE,aAAa;IACxB,MAAM,EAAE,UAAU;IAClB,MAAM,EAAE,UAAU;IAClB,IAAI,EAAE,QAAQ;CACf,CAAC;AAEF,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,KAAuB;IAEvB,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC/B,OAAO,WAAW,CAAC,KAAK,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;IACxC,CAAC;IAED,MAAM,GAAG,GAAG,SAAS,EAAE,CAAC;IACxB,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM,IAAI,EAAE,SAAS,EAAE,GAAG,CAAC,qBAAqB,EAAE,CAAC;IACxE,IAAI,MAAM,CAAC,SAAS,IAAI,CAAC,EAAE,CAAC;QAC1B,OAAO,WAAW,CAAC,KAAK,CAAC,OAAO,EAAE;YAChC,2DAA2D,GAAG,CAAC,qBAAqB,mDAAmD;SACxI,CAAC,CAAC;IACL,CAAC;IAED,MAAM,QAAQ,GAAG,cAAc,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IAC7C,IAAI,CAAC,QAAQ,EAAE,CAAC;QACd,MAAM,OAAO,GAAG,YAAY,EAAE;aAC3B,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC;aAC7B,IAAI,CAAC,IAAI,CAAC,CAAC;QACd,OAAO,WAAW,CAAC,KAAK,CAAC,OAAO,EAAE;YAChC,8CAA8C,OAAO,KAAK;gBACxD,GAAG,KAAK,CAAC,OAAO,CAAC,MAAM,qCAAqC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG;SAC1F,CAAC,CAAC;IACL,CAAC;IAED,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,QAAQ,CAAC,CAAW,CAAC;IAC/D,MAAM,MAAM,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC;IAClC,MAAM,UAAU,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC;IACtC,MAAM,UAAU,GAAG,UAAU,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;IAChD,MAAM,OAAO,GAAG,GAAG,QAAQ,UAAU,CAAC;IAEtC,mBAAmB,EAAE,CAAC;IACtB,MAAM,MAAM,GAAG,cAAc,CAAC,OAAO,EAAE,UAAU,EAAE,UAAU,CAAC,CAAC;IAC/D,IAAI,MAAM,EAAE,CAAC;QACX,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAA4B,CAAC;QAC7D,OAAO;YACL,MAAM,EAAE,gBAAgB,CAAC,KAAK,CAAC,OAAO,EAAE,MAAM,EAAE,KAAK,CAAC,OAAO,CAAC;YAC9D,QAAQ;YACR,KAAK,EAAE,OAAO;YACd,MAAM,EAAE,IAAI;YACZ,SAAS,EAAE,CAAC;YACZ,QAAQ,EAAE,EAAE;SACb,CAAC;IACJ,CAAC;IAED,IAAI,MAAwB,CAAC;IAC7B,IAAI,CAAC;QACH,MAAM,GAAG,MAAM,QAAQ,CAAC,QAAQ,CAAC,CAC/B,EAAE,MAAM,EAAE,UAAU,EAAE,KAAK,CAAC,UAAU,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,EAC9D,MAAM,CACP,CAAC;IACJ,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,OAAO,WAAW,CAAC,KAAK,CAAC,OAAO,EAAE;YAChC,iBAAiB,QAAQ,aAAc,CAAW,CAAC,OAAO,EAAE;SAC7D,CAAC,CAAC;IACL,CAAC;YAAS,CAAC;QACT,MAAM,CAAC,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,CAAC,SAAS,GAAG,CAAC,CAAC,CAAC;IACvD,CAAC;IAED,MAAM,MAAM,GAAG,qBAAqB,CAAC,MAAM,CAAC,MAAM,EAAE,KAAK,CAAC,UAAU,CAAC,CAAC;IACtE,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACtB,OAAO,WAAW,CAAC,KAAK,CAAC,OAAO,EAAE;YAChC,iBAAiB,QAAQ,wCAAwC,MAAM;iBACpE,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,OAAO,EAAE,CAAC;iBACpC,IAAI,CAAC,IAAI,CAAC,EAAE;SAChB,CAAC,CAAC;IACL,CAAC;IAED,MAAM,KAAK,GAAG,GAAG,CAAC,eAAe,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC;IACxD,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACvB,cAAc,CAAC;QACb,OAAO;QACP,UAAU;QACV,UAAU;QACV,QAAQ,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,MAAM,CAAC;QACvC,SAAS,EAAE,GAAG;QACd,SAAS,EAAE,GAAG,GAAG,KAAK;KACvB,CAAC,CAAC;IAEH,OAAO;QACL,MAAM,EAAE,gBAAgB,CAAC,KAAK,CAAC,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,KAAK,CAAC,OAAO,CAAC;QACrE,QAAQ;QACR,KAAK,EAAE,MAAM,CAAC,KAAK;QACnB,MAAM,EAAE,KAAK;QACb,SAAS,EAAE,MAAM,CAAC,SAAS;QAC3B,QAAQ,EAAE,MAAM,CAAC,QAAQ,IAAI,EAAE;KAChC,CAAC;AACJ,CAAC;AAED,SAAS,WAAW,CAClB,OAAgC,EAChC,QAAkB;IAElB,OAAO;QACL,MAAM,EAAE,EAAE,GAAG,OAAO,EAAE;QACtB,QAAQ,EAAE,WAAW;QACrB,KAAK,EAAE,EAAE;QACT,MAAM,EAAE,KAAK;QACb,SAAS,EAAE,CAAC;QACZ,QAAQ;KACT,CAAC;AACJ,CAAC;AAED,SAAS,gBAAgB,CACvB,OAAgC,EAChC,MAA+B,EAC/B,OAAiB;IAEjB,MAAM,GAAG,GAAG,EAAE,GAAG,OAAO,EAAE,CAAC;IAC3B,KAAK,MAAM,GAAG,IAAI,OAAO,EAAE,CAAC;QAC1B,IAAI,MAAM,CAAC,GAAG,CAAC,KAAK,SAAS;YAAE,GAAG,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC;IACxD,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,SAAS,WAAW,CAAC,KAAuB;IAC1C,MAAM,IAAI,GAAG,QAAQ,CAAC,KAAK,CAAC,IAAI,EAAE,cAAc,CAAC,CAAC;IAClD,OAAO;QACL,2DAA2D;QAC3D,mBAAmB,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG;QAC9C,iHAAiH;QACjH,EAAE;QACF,OAAO;QACP,IAAI;KACL,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AACf,CAAC;AAED,SAAS,QAAQ,CAAC,CAAS,EAAE,QAAgB;IAC3C,IAAI,CAAC,CAAC,MAAM,IAAI,QAAQ;QAAE,OAAO,CAAC,CAAC;IACnC,OAAO,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC;AAC9B,CAAC"}

package/dist/extraction/markdown.d.ts

@@ -1,4 +1,13 @@
+import TurndownService from 'turndown';
+export declare function buildTurndown(): TurndownService;
 export declare function htmlToMarkdown(html: string): string;
+export interface Heading {
+    level: number;
+    text: string;
+    lineIndex: number;
+}
+export declare function parseHeadings(lines: string[]): Heading[];
+export declare function lineStartCharOffsets(lines: string[]): number[];
 export declare function extractSection(markdown: string, section: string, sectionIndex?: number): {
     content: string;
     matched: boolean;

package/dist/extraction/markdown.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"markdown.d.ts","sourceRoot":"","sources":["../../src/extraction/markdown.ts"],"names":[],"mappings":"AAkDA,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAGnD;AAmCD,wBAAgB,cAAc,CAC5B,QAAQ,EAAE,MAAM,EAChB,OAAO,EAAE,MAAM,EACf,YAAY,SAAI,GACf;IAAE,OAAO,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,OAAO,CAAA;CAAE,CA2BvC;AAED,wBAAgB,qBAAqB,CAAC,QAAQ,EAAE,MAAM,GAAG;IAAE,KAAK,EAAE,MAAM,EAAE,CAAC;IAAC,MAAM,EAAE,MAAM,EAAE,CAAA;CAAE,CAoB7F;AAkBD,wBAAgB,sBAAsB,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,CAsB/D;AAID,wBAAgB,mBAAmB,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,MAAM,CAkC7E"}
+{"version":3,"file":"markdown.d.ts","sourceRoot":"","sources":["../../src/extraction/markdown.ts"],"names":[],"mappings":"AAAA,OAAO,eAAe,MAAM,UAAU,CAAC;AAiBvC,wBAAgB,aAAa,IAAI,eAAe,CA2D/C;AAID,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAGnD;AAED,MAAM,WAAW,OAAO;IACtB,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,wBAAgB,aAAa,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,OAAO,EAAE,CASxD;AAID,wBAAgB,oBAAoB,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,MAAM,EAAE,CAQ9D;AAkBD,wBAAgB,cAAc,CAC5B,QAAQ,EAAE,MAAM,EAChB,OAAO,EAAE,MAAM,EACf,YAAY,SAAI,GACf;IAAE,OAAO,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,OAAO,CAAA;CAAE,CA2BvC;AAED,wBAAgB,qBAAqB,CAAC,QAAQ,EAAE,MAAM,GAAG;IAAE,KAAK,EAAE,MAAM,EAAE,CAAC;IAAC,MAAM,EAAE,MAAM,EAAE,CAAA;CAAE,CAoB7F;AAkBD,wBAAgB,sBAAsB,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,CAsB/D;AAID,wBAAgB,mBAAmB,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,MAAM,CAyC7E"}

package/dist/extraction/markdown.js

@@ -1,5 +1,21 @@
 import TurndownService from 'turndown';
-function buildTurndown() {
+import { detectCodeLanguage } from './lang-hints.js';
+function longestBacktickRun(s) {
+    let max = 0;
+    let cur = 0;
+    for (let i = 0; i < s.length; i++) {
+        if (s.charCodeAt(i) === 96) {
+            cur++;
+            if (cur > max)
+                max = cur;
+        }
+        else {
+            cur = 0;
+        }
+    }
+    return max;
+}
+export function buildTurndown() {
     const td = new TurndownService({ headingStyle: 'atx', codeBlockStyle: 'fenced' });
     // Remove script and style tags entirely
     td.remove(['script', 'style']);
@@ -35,6 +51,20 @@ function buildTurndown() {
             return content;
         },
     });
+    td.addRule('codeBlockLang', {
+        filter(node) {
+            return node.nodeName === 'PRE' && node.querySelector('code') !== null;
+        },
+        replacement(_content, node) {
+            const pre = node;
+            const code = pre.querySelector('code');
+            const cls = code?.getAttribute('class') ?? pre.getAttribute('class') ?? '';
+            const lang = detectCodeLanguage(cls);
+            const body = code?.textContent ?? pre.textContent ?? '';
+            const fence = '`'.repeat(Math.max(3, longestBacktickRun(body) + 1));
+            return `\n\n${fence}${lang ?? ''}\n${body.replace(/\n+$/, '')}\n${fence}\n\n`;
+        },
+    });
     return td;
 }
 const turndown = buildTurndown();
@@ -43,7 +73,7 @@ export function htmlToMarkdown(html) {
         return '';
     return turndown.turndown(html);
 }
-function parseHeadings(lines) {
+export function parseHeadings(lines) {
     const headings = [];
     for (let i = 0; i < lines.length; i++) {
         const match = lines[i].match(/^(#{1,6})\s+(.+)/);
@@ -53,6 +83,17 @@ function parseHeadings(lines) {
     }
     return headings;
 }
+// Prefix-sum array of char offsets: offsets[i] is the index in
+// `lines.join('\n')` at which lines[i] begins.
+export function lineStartCharOffsets(lines) {
+    const offsets = new Array(lines.length);
+    let acc = 0;
+    for (let i = 0; i < lines.length; i++) {
+        offsets[i] = acc;
+        acc += lines[i].length + 1; // +1 for the '\n' separator
+    }
+    return offsets;
+}
 function extractFromHeading(lines, headings, headingIdx) {
     const heading = headings[headingIdx];
     const start = heading.lineIndex;
@@ -151,8 +192,16 @@ export function resolveRelativeUrls(markdown, baseUrl) {
         const trimmed = path.trim();
         if (!trimmed)
             return path;
-        if (/^(?:https?:|mailto:|tel:|javascript:|data:|#)/i.test(trimmed))
+        if (/^(?:https?:|mailto:|tel:|javascript:|data:)/i.test(trimmed))
             return path;
+        if (trimmed.startsWith('#')) {
+            try {
+                return new URL(trimmed, baseUrl).href;
+            }
+            catch {
+                return path;
+            }
+        }
         if (trimmed.startsWith('//')) {
             try {
                 const base = new URL(baseUrl);