greptor 0.2.1

package/dist/metadata-schema/initialize.js

@@ -0,0 +1,37 @@
+import { mkdir, readFile, writeFile } from "node:fs/promises";
+import path from "node:path";
+import YAML from "yaml";
+import { fileExists } from "../utils/file.js";
+import { generateMetadataSchema } from "./generate.js";
+export const METADATA_SCHEMA_FILENAME = "metadata-schema.yaml";
+async function persist(schemaFilePath, metadataSchema, logger) {
+    const schemaYaml = YAML.stringify(metadataSchema);
+    await mkdir(path.dirname(schemaFilePath), { recursive: true });
+    await writeFile(schemaFilePath, schemaYaml, "utf8");
+    logger?.debug?.("Metadata schema saved", { path: schemaFilePath });
+}
+export async function initializeMetadataSchema(baseDir, llmModel, topic, metadataSchema, logger) {
+    const schemaFilePath = path.join(baseDir, METADATA_SCHEMA_FILENAME);
+    // If a schema is provided, save it to disk and return it straight away
+    if (metadataSchema) {
+        await persist(schemaFilePath, metadataSchema, logger);
+        return metadataSchema;
+    }
+    // If schema file exists on disk, load and return it
+    if (await fileExists(schemaFilePath)) {
+        logger?.debug?.("Metadata schema not provided, loading from file", {
+            path: schemaFilePath,
+        });
+        return YAML.parse(await readFile(schemaFilePath, "utf8"));
+    }
+    // Otherwise, generate a new schema using the LLM
+    logger?.info?.("Generating metadata schema", { topic });
+    const schema = await generateMetadataSchema(topic, llmModel);
+    await persist(schemaFilePath, schema, logger);
+    logger?.info?.("Metadata schema generated", {
+        path: schemaFilePath,
+        fields: schema.length,
+    });
+    return schema;
+}
+//# sourceMappingURL=initialize.js.map

package/dist/metadata-schema/initialize.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"initialize.js","sourceRoot":"","sources":["../../src/metadata-schema/initialize.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAC9D,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,IAAI,MAAM,MAAM,CAAC;AAGxB,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAC9C,OAAO,EAAE,sBAAsB,EAAE,MAAM,eAAe,CAAC;AAEvD,MAAM,CAAC,MAAM,wBAAwB,GAAG,sBAAsB,CAAC;AAE/D,KAAK,UAAU,OAAO,CACrB,cAAsB,EACtB,cAA8B,EAC9B,MAAe;IAEf,MAAM,UAAU,GAAG,IAAI,CAAC,SAAS,CAAC,cAAc,CAAC,CAAC;IAClD,MAAM,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAC/D,MAAM,SAAS,CAAC,cAAc,EAAE,UAAU,EAAE,MAAM,CAAC,CAAC;IAEpD,MAAM,EAAE,KAAK,EAAE,CAAC,uBAAuB,EAAE,EAAE,IAAI,EAAE,cAAc,EAAE,CAAC,CAAC;AACpE,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,wBAAwB,CAC7C,OAAe,EACf,QAAgB,EAChB,KAAa,EACb,cAA+B,EAC/B,MAAe;IAEf,MAAM,cAAc,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,wBAAwB,CAAC,CAAC;IAEpE,uEAAuE;IACvE,IAAI,cAAc,EAAE,CAAC;QACpB,MAAM,OAAO,CAAC,cAAc,EAAE,cAAc,EAAE,MAAM,CAAC,CAAC;QACtD,OAAO,cAAc,CAAC;IACvB,CAAC;IAED,oDAAoD;IACpD,IAAI,MAAM,UAAU,CAAC,cAAc,CAAC,EAAE,CAAC;QACtC,MAAM,EAAE,KAAK,EAAE,CAAC,iDAAiD,EAAE;YAClE,IAAI,EAAE,cAAc;SACpB,CAAC,CAAC;QAEH,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC,CAAmB,CAAC;IAC7E,CAAC;IAED,iDAAiD;IACjD,MAAM,EAAE,IAAI,EAAE,CAAC,4BAA4B,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC;IACxD,MAAM,MAAM,GAAG,MAAM,sBAAsB,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC;IAC7D,MAAM,OAAO,CAAC,cAAc,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;IAC9C,MAAM,EAAE,IAAI,EAAE,CAAC,2BAA2B,EAAE;QAC3C,IAAI,EAAE,cAAc;QACpB,MAAM,EAAE,MAAM,CAAC,MAAM;KACrB,CAAC,CAAC;IAEH,OAAO,MAAM,CAAC;AACf,CAAC"}

package/dist/metadata-schema/types.d.ts

@@ -0,0 +1,34 @@
+import { z } from "zod";
+export declare const MetadataFieldSchema: z.ZodObject<{
+    name: z.ZodString;
+    type: z.ZodEnum<{
+        string: "string";
+        number: "number";
+        boolean: "boolean";
+        "string[]": "string[]";
+        "number[]": "number[]";
+        enum: "enum";
+        "enum[]": "enum[]";
+        date: "date";
+    }>;
+    description: z.ZodString;
+    enumValues: z.ZodNullable<z.ZodOptional<z.ZodArray<z.ZodString>>>;
+}, z.core.$strip>;
+export declare const ResponseSchema: z.ZodObject<{
+    metadata_fields: z.ZodArray<z.ZodObject<{
+        name: z.ZodString;
+        type: z.ZodEnum<{
+            string: "string";
+            number: "number";
+            boolean: "boolean";
+            "string[]": "string[]";
+            "number[]": "number[]";
+            enum: "enum";
+            "enum[]": "enum[]";
+            date: "date";
+        }>;
+        description: z.ZodString;
+        enumValues: z.ZodNullable<z.ZodOptional<z.ZodArray<z.ZodString>>>;
+    }, z.core.$strip>>;
+}, z.core.$strip>;
+//# sourceMappingURL=types.d.ts.map

package/dist/metadata-schema/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/metadata-schema/types.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,eAAO,MAAM,mBAAmB;;;;;;;;;;;;;;iBAoB9B,CAAC;AAEH,eAAO,MAAM,cAAc;;;;;;;;;;;;;;;;iBAMzB,CAAC"}

package/dist/metadata-schema/types.js

@@ -0,0 +1,30 @@
+import { z } from "zod";
+export const MetadataFieldSchema = z.object({
+    name: z.string().describe("Metadata field name in snake_case"),
+    type: z
+        .enum([
+        "string",
+        "string[]",
+        "number",
+        "number[]",
+        "boolean",
+        "enum",
+        "enum[]",
+        "date",
+    ])
+        .describe("Field data type"),
+    description: z.string().describe("Purpose and usage of this metadata field"),
+    enumValues: z
+        .array(z.string())
+        .optional()
+        .nullable()
+        .describe("Full list of enum values for enum types."),
+});
+export const ResponseSchema = z.object({
+    metadata_fields: z
+        .array(MetadataFieldSchema)
+        .min(5)
+        .max(10)
+        .describe("List of metadata fields for the given topic"),
+});
+//# sourceMappingURL=types.js.map

package/dist/metadata-schema/types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/metadata-schema/types.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,MAAM,CAAC,MAAM,mBAAmB,GAAG,CAAC,CAAC,MAAM,CAAC;IAC3C,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,mCAAmC,CAAC;IAC9D,IAAI,EAAE,CAAC;SACL,IAAI,CAAC;QACL,QAAQ;QACR,UAAU;QACV,QAAQ;QACR,UAAU;QACV,SAAS;QACT,MAAM;QACN,QAAQ;QACR,MAAM;KACN,CAAC;SACD,QAAQ,CAAC,iBAAiB,CAAC;IAC7B,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,0CAA0C,CAAC;IAC5E,UAAU,EAAE,CAAC;SACX,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;SACjB,QAAQ,EAAE;SACV,QAAQ,EAAE;SACV,QAAQ,CAAC,0CAA0C,CAAC;CACtD,CAAC,CAAC;AAEH,MAAM,CAAC,MAAM,cAAc,GAAG,CAAC,CAAC,MAAM,CAAC;IACtC,eAAe,EAAE,CAAC;SAChB,KAAK,CAAC,mBAAmB,CAAC;SAC1B,GAAG,CAAC,CAAC,CAAC;SACN,GAAG,CAAC,EAAE,CAAC;SACP,QAAQ,CAAC,6CAA6C,CAAC;CACzD,CAAC,CAAC"}

package/dist/processing/chunk.d.ts

@@ -0,0 +1,3 @@
+import type { LlmClient } from "../llm/llm-factory.js";
+export declare function chunk(rawContent: string, domain: string, llm: LlmClient): Promise<string>;
+//# sourceMappingURL=chunk.d.ts.map

package/dist/processing/chunk.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"chunk.d.ts","sourceRoot":"","sources":["../../src/processing/chunk.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,uBAAuB,CAAC;AAuBvD,wBAAsB,KAAK,CAC1B,UAAU,EAAE,MAAM,EAClB,MAAM,EAAE,MAAM,EACd,GAAG,EAAE,SAAS,GACZ,OAAO,CAAC,MAAM,CAAC,CAiBjB"}

package/dist/processing/chunk.js

@@ -0,0 +1,36 @@
+const createCleanPrompt = (text, domain) => `
+Clean + structure the raw content into independent semantic chunks for domain: ${domain}
+
+Remove completely (noise/boilerplate): ads/sponsors, greetings/intros/outros, CTAs/promos ("like & subscribe"), duplicates/filler, contact/social/discount codes, sign-offs ("reach out", "visit our website"), anything not primary informational content.
+
+Preserve meaning exactly: no summarizing/paraphrasing. Keep all facts/numbers/names/dates, domain terminology, tables/lists, and URLs only if meaningful.
+
+Normalize: canonicalize entity names; remove formatting garbage/repeated boilerplate; normalize whitespace/tabs/newlines.
+
+Chunking: prefer fewer richer chunks. Target 100–200+ words per chunk where possible. Group related subtopics; split only when topics are truly distinct or context switches.
+
+Output (English only), chunks separated by blank lines:
+CHUNK c01: "Short Topic Title"
+<cleaned content>
+
+Rules: do not paraphrase, shorten meaning, add interpretation, or merge unrelated topics.
+
+RAW CONTENT:
+${text}
+`;
+export async function chunk(rawContent, domain, llm) {
+    const prompt = createCleanPrompt(rawContent, domain);
+    const messages = [
+        { role: "user", content: prompt },
+    ];
+    const completion = await llm.client.chat.completions.parse({
+        model: llm.model,
+        messages,
+    });
+    const content = completion.choices[0]?.message?.content;
+    if (!content) {
+        throw new Error("Failed to clean content: empty LLM response");
+    }
+    return content;
+}
+//# sourceMappingURL=chunk.js.map

package/dist/processing/chunk.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"chunk.js","sourceRoot":"","sources":["../../src/processing/chunk.ts"],"names":[],"mappings":"AAGA,MAAM,iBAAiB,GAAG,CAAC,IAAY,EAAE,MAAc,EAAE,EAAE,CAAC;iFACqB,MAAM;;;;;;;;;;;;;;;;;EAiBrF,IAAI;CACL,CAAC;AAEF,MAAM,CAAC,KAAK,UAAU,KAAK,CAC1B,UAAkB,EAClB,MAAc,EACd,GAAc;IAEd,MAAM,MAAM,GAAG,iBAAiB,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC;IACrD,MAAM,QAAQ,GAAiC;QAC9C,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE;KACjC,CAAC;IAEF,MAAM,UAAU,GAAG,MAAM,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC;QAC1D,KAAK,EAAE,GAAG,CAAC,KAAK;QAChB,QAAQ;KACR,CAAC,CAAC;IAEH,MAAM,OAAO,GAAG,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,OAAO,CAAC;IACxD,IAAI,CAAC,OAAO,EAAE,CAAC;QACd,MAAM,IAAI,KAAK,CAAC,6CAA6C,CAAC,CAAC;IAChE,CAAC;IAED,OAAO,OAAO,CAAC;AAChB,CAAC"}

package/dist/processing/extract-metadata.d.ts

@@ -0,0 +1,4 @@
+import type { LlmClient } from "../llm/llm-factory.js";
+import type { Metadata } from "../types.js";
+export declare function extractMetadata(chunkedContent: string, domain: string, metadataSchema: string, llm: LlmClient): Promise<Metadata[]>;
+//# sourceMappingURL=extract-metadata.d.ts.map

package/dist/processing/extract-metadata.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"extract-metadata.d.ts","sourceRoot":"","sources":["../../src/processing/extract-metadata.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,uBAAuB,CAAC;AACvD,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AA6B5C,wBAAsB,eAAe,CACpC,cAAc,EAAE,MAAM,EACtB,MAAM,EAAE,MAAM,EACd,cAAc,EAAE,MAAM,EACtB,GAAG,EAAE,SAAS,GACZ,OAAO,CAAC,QAAQ,EAAE,CAAC,CAqBrB"}

package/dist/processing/extract-metadata.js

@@ -0,0 +1,39 @@
+import YAML from "yaml";
+const createExtractMetadataPrompt = (text, domain, metadataSchema) => `
+Extract per-chunk metadata for domain: ${domain}. Optimize for grep-based search/filtering.
+
+Input format:
+CHUNK c01: "Title"\n...\n\nCHUNK c02: "Title"\n...
+
+Output: YAML ONLY (no fences/comments). YAML list like:
+- id: c01
+	title: "Title"
+	key1: value
+	key2: [v1, v2]
+
+Value formats: strings snake_case (except title); numbers put unit in key (price_aud: 1250000); percentages use _percent; dates ISO-8601; ranges as [min, max]; arrays MUST be single-line YAML (e.g., [a, b]).
+
+Rules: extract each chunk separately; do not output null/empty fields; for enum fields, only use values from schema; include additional useful numeric/date metrics for grep filtering.
+
+SCHEMA:
+${metadataSchema}
+
+INPUT:
+${text}
+`;
+export async function extractMetadata(chunkedContent, domain, metadataSchema, llm) {
+    const prompt = createExtractMetadataPrompt(chunkedContent, domain, metadataSchema);
+    const messages = [
+        { role: "user", content: prompt },
+    ];
+    const completion = await llm.client.chat.completions.parse({
+        model: llm.model,
+        messages,
+    });
+    const content = completion.choices[0]?.message?.content;
+    if (!content) {
+        throw new Error("Failed to extract metadata: empty LLM response");
+    }
+    return YAML.parse(content);
+}
+//# sourceMappingURL=extract-metadata.js.map

package/dist/processing/extract-metadata.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"extract-metadata.js","sourceRoot":"","sources":["../../src/processing/extract-metadata.ts"],"names":[],"mappings":"AACA,OAAO,IAAI,MAAM,MAAM,CAAC;AAIxB,MAAM,2BAA2B,GAAG,CACnC,IAAY,EACZ,MAAc,EACd,cAAsB,EACrB,EAAE,CAAC;yCACoC,MAAM;;;;;;;;;;;;;;;;EAgB7C,cAAc;;;EAGd,IAAI;CACL,CAAC;AAEF,MAAM,CAAC,KAAK,UAAU,eAAe,CACpC,cAAsB,EACtB,MAAc,EACd,cAAsB,EACtB,GAAc;IAEd,MAAM,MAAM,GAAG,2BAA2B,CACzC,cAAc,EACd,MAAM,EACN,cAAc,CACd,CAAC;IACF,MAAM,QAAQ,GAAiC;QAC9C,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE;KACjC,CAAC;IAEF,MAAM,UAAU,GAAG,MAAM,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC;QAC1D,KAAK,EAAE,GAAG,CAAC,KAAK;QAChB,QAAQ;KACR,CAAC,CAAC;IAEH,MAAM,OAAO,GAAG,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,OAAO,CAAC;IACxD,IAAI,CAAC,OAAO,EAAE,CAAC;QACd,MAAM,IAAI,KAAK,CAAC,gDAAgD,CAAC,CAAC;IACnE,CAAC;IAED,OAAO,IAAI,CAAC,KAAK,CAAC,OAAO,CAAe,CAAC;AAC1C,CAAC"}

package/dist/processing/processor.d.ts

@@ -0,0 +1,28 @@
+import type { LlmClient } from "../llm/llm-factory.js";
+import type { DocumentRef, FileStorage } from "../storage/index.js";
+import type { Logger } from "../types.js";
+export interface ProcessorContext {
+    domain: string;
+    metadataSchema: string;
+    llm: LlmClient;
+    storage: FileStorage;
+    logger?: Logger;
+}
+export interface ProcessingQueue {
+    enqueue: (ref: DocumentRef) => void;
+    dequeue: () => DocumentRef | undefined;
+    size: () => number;
+}
+export declare function createProcessingQueue(): ProcessingQueue;
+export declare function startBackgroundWorkers(args: {
+    ctx: ProcessorContext;
+    queue: ProcessingQueue;
+    concurrency?: number;
+    idleSleepMs?: number;
+}): void;
+export declare function enqueueUnprocessedDocuments(args: {
+    storage: FileStorage;
+    queue: ProcessingQueue;
+    logger?: Logger;
+}): Promise<number>;
+//# sourceMappingURL=processor.d.ts.map

package/dist/processing/processor.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"processor.d.ts","sourceRoot":"","sources":["../../src/processing/processor.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,uBAAuB,CAAC;AACvD,OAAO,KAAK,EAAE,WAAW,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AACpE,OAAO,KAAK,EAAE,MAAM,EAAY,MAAM,aAAa,CAAC;AAMpD,MAAM,WAAW,gBAAgB;IAChC,MAAM,EAAE,MAAM,CAAC;IACf,cAAc,EAAE,MAAM,CAAC;IACvB,GAAG,EAAE,SAAS,CAAC;IACf,OAAO,EAAE,WAAW,CAAC;IACrB,MAAM,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,eAAe;IAC/B,OAAO,EAAE,CAAC,GAAG,EAAE,WAAW,KAAK,IAAI,CAAC;IACpC,OAAO,EAAE,MAAM,WAAW,GAAG,SAAS,CAAC;IACvC,IAAI,EAAE,MAAM,MAAM,CAAC;CACnB;AAED,wBAAgB,qBAAqB,IAAI,eAAe,CAgBvD;AAgFD,wBAAgB,sBAAsB,CAAC,IAAI,EAAE;IAC5C,GAAG,EAAE,gBAAgB,CAAC;IACtB,KAAK,EAAE,eAAe,CAAC;IACvB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,WAAW,CAAC,EAAE,MAAM,CAAC;CACrB,GAAG,IAAI,CAkCP;AAED,wBAAsB,2BAA2B,CAAC,IAAI,EAAE;IACvD,OAAO,EAAE,WAAW,CAAC;IACrB,KAAK,EAAE,eAAe,CAAC;IACvB,MAAM,CAAC,EAAE,MAAM,CAAC;CAChB,GAAG,OAAO,CAAC,MAAM,CAAC,CAalB"}

package/dist/processing/processor.js

@@ -0,0 +1,112 @@
+import YAML from "yaml";
+import { chunk as chunkDocument } from "./chunk.js";
+import { extractMetadata } from "./extract-metadata.js";
+const DEFAULT_IDLE_SLEEP_MS = 750;
+export function createProcessingQueue() {
+    const items = [];
+    return {
+        enqueue(ref) {
+            items.push(ref);
+        },
+        size() {
+            return items.length;
+        },
+        dequeue() {
+            return items.shift();
+        },
+    };
+}
+function renderProcessedDocument(metadata, chunkMetadata, chunkContent) {
+    const combinedMetadata = {
+        ...metadata,
+        chunks: chunkMetadata,
+    };
+    const doc = new YAML.Document(combinedMetadata);
+    YAML.visit(doc, {
+        Seq(_, node) {
+            const allScalars = node.items.every((item) => YAML.isScalar(item));
+            if (allScalars) {
+                node.flow = true;
+            }
+        },
+    });
+    const renderedMetadata = doc.toString({ lineWidth: 200 });
+    return [
+        "---",
+        renderedMetadata.trimEnd(),
+        "---",
+        "",
+        chunkContent.trim(),
+    ].join("\n");
+}
+async function processDocument(ref, ctx) {
+    // 1. Read raw content
+    const { metadata, content } = await ctx.storage.readRawContent(ref);
+    const contentLength = content.length;
+    // 2. Chunk content with LLM
+    ctx.logger?.debug?.("Chunking document", { ref, step: "chunk" });
+    const chunkContent = await chunkDocument(content, ctx.domain, ctx.llm);
+    // 3. Extract metadata with LLM
+    ctx.logger?.debug?.("Extracting metadata", { ref, step: "metadata" });
+    const chunkMetadata = await extractMetadata(chunkContent, ctx.domain, ctx.metadataSchema, ctx.llm);
+    // 4. Parse chunk metadata and render final content
+    const rendered = renderProcessedDocument(metadata, chunkMetadata, chunkContent);
+    // 5. Save processed content
+    await ctx.storage.saveProcessedContent(ref, rendered);
+    ctx.logger?.info?.("Document processed", {
+        ref,
+        chunks: chunkMetadata.length,
+        bytes: contentLength,
+    });
+}
+function sleep(ms) {
+    return new Promise((resolve) => {
+        const t = setTimeout(resolve, ms);
+        // If nothing else is keeping the process alive, don't block exit.
+        t.unref?.();
+    });
+}
+export function startBackgroundWorkers(args) {
+    const concurrency = Math.max(1, args.concurrency ?? 1);
+    const idleSleepMs = Math.max(50, args.idleSleepMs ?? DEFAULT_IDLE_SLEEP_MS);
+    const { ctx, queue } = args;
+    async function workerLoop(workerIndex) {
+        while (true) {
+            const docRef = queue.dequeue();
+            if (!docRef) {
+                await sleep(idleSleepMs);
+                continue;
+            }
+            ctx.logger?.debug?.("Processing started", {
+                worker: workerIndex,
+                ref: docRef,
+            });
+            try {
+                await processDocument(docRef, ctx);
+            }
+            catch (error) {
+                ctx.logger?.error?.("Processing failed", {
+                    err: error,
+                    ref: docRef,
+                    worker: workerIndex,
+                });
+            }
+        }
+    }
+    for (let i = 0; i < concurrency; i++) {
+        workerLoop(i + 1);
+    }
+    ctx.logger?.debug?.("Background workers started", { concurrency });
+}
+export async function enqueueUnprocessedDocuments(args) {
+    const refs = await args.storage.getUnprocessedContents();
+    for (const ref of refs) {
+        args.logger?.debug?.("Queued unprocessed document", { ref });
+        args.queue.enqueue(ref);
+    }
+    if (refs.length > 0) {
+        args.logger?.debug?.("Found unprocessed documents", { count: refs.length });
+    }
+    return refs.length;
+}
+//# sourceMappingURL=processor.js.map

package/dist/processing/processor.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"processor.js","sourceRoot":"","sources":["../../src/processing/processor.ts"],"names":[],"mappings":"AAAA,OAAO,IAAI,MAAM,MAAM,CAAC;AAIxB,OAAO,EAAE,KAAK,IAAI,aAAa,EAAE,MAAM,YAAY,CAAC;AACpD,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AAExD,MAAM,qBAAqB,GAAG,GAAG,CAAC;AAgBlC,MAAM,UAAU,qBAAqB;IACpC,MAAM,KAAK,GAAkB,EAAE,CAAC;IAEhC,OAAO;QACN,OAAO,CAAC,GAAG;YACV,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACjB,CAAC;QAED,IAAI;YACH,OAAO,KAAK,CAAC,MAAM,CAAC;QACrB,CAAC;QAED,OAAO;YACN,OAAO,KAAK,CAAC,KAAK,EAAE,CAAC;QACtB,CAAC;KACD,CAAC;AACH,CAAC;AAED,SAAS,uBAAuB,CAC/B,QAAkB,EAClB,aAAyB,EACzB,YAAoB;IAEpB,MAAM,gBAAgB,GAAG;QACxB,GAAG,QAAQ;QACX,MAAM,EAAE,aAAa;KACrB,CAAC;IAEF,MAAM,GAAG,GAAG,IAAI,IAAI,CAAC,QAAQ,CAAC,gBAAgB,CAAC,CAAC;IAEhD,IAAI,CAAC,KAAK,CAAC,GAAG,EAAE;QACf,GAAG,CAAC,CAAC,EAAE,IAAI;YACV,MAAM,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC;YACnE,IAAI,UAAU,EAAE,CAAC;gBAChB,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;YAClB,CAAC;QACF,CAAC;KACD,CAAC,CAAC;IAEH,MAAM,gBAAgB,GAAG,GAAG,CAAC,QAAQ,CAAC,EAAE,SAAS,EAAE,GAAG,EAAE,CAAC,CAAC;IAE1D,OAAO;QACN,KAAK;QACL,gBAAgB,CAAC,OAAO,EAAE;QAC1B,KAAK;QACL,EAAE;QACF,YAAY,CAAC,IAAI,EAAE;KACnB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AACd,CAAC;AAED,KAAK,UAAU,eAAe,CAC7B,GAAgB,EAChB,GAAqB;IAErB,sBAAsB;IACtB,MAAM,EAAE,QAAQ,EAAE,OAAO,EAAE,GAAG,MAAM,GAAG,CAAC,OAAO,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC;IACpE,MAAM,aAAa,GAAG,OAAO,CAAC,MAAM,CAAC;IAErC,4BAA4B;IAC5B,GAAG,CAAC,MAAM,EAAE,KAAK,EAAE,CAAC,mBAAmB,EAAE,EAAE,GAAG,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC,CAAC;IACjE,MAAM,YAAY,GAAG,MAAM,aAAa,CAAC,OAAO,EAAE,GAAG,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,CAAC,CAAC;IAEvE,+BAA+B;IAC/B,GAAG,CAAC,MAAM,EAAE,KAAK,EAAE,CAAC,qBAAqB,EAAE,EAAE,GAAG,EAAE,IAAI,EAAE,UAAU,EAAE,CAAC,CAAC;IACtE,MAAM,aAAa,GAAG,MAAM,eAAe,CAC1C,YAAY,EACZ,GAAG,CAAC,MAAM,EACV,GAAG,CAAC,cAAc,EAClB,GAAG,CAAC,GAAG,CACP,CAAC;IAEF,mDAAmD;IACnD,MAAM,QAAQ,GAAG,uBAAuB,CACvC,QAAQ,EACR,aAAa,EACb,YAAY,CACZ,CAAC;IAEF,4BAA4B;IAC5B,MAAM,GAAG,CAAC,OAAO,CAAC,oBAAoB,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;IAEtD,GAAG,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,oBAAoB,EAAE;QACxC,GAAG;QACH,MAAM,EAAE,aAAa,CAAC,MAAM;QAC5B,KAAK,EAAE,aAAa;KACpB,CAAC,CAAC;AACJ,CAAC;AAED,SAAS,KAAK,CAAC,EAAU;IACxB,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE;QAC9B,MAAM,CAAC,GAAG,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;QAClC,kEAAkE;QACjE,CAAuC,CAAC,KAAK,EAAE,EAAE,CAAC;IACpD,CAAC,CAAC,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,sBAAsB,CAAC,IAKtC;IACA,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,WAAW,IAAI,CAAC,CAAC,CAAC;IACvD,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,IAAI,CAAC,WAAW,IAAI,qBAAqB,CAAC,CAAC;IAC5E,MAAM,EAAE,GAAG,EAAE,KAAK,EAAE,GAAG,IAAI,CAAC;IAE5B,KAAK,UAAU,UAAU,CAAC,WAAmB;QAC5C,OAAO,IAAI,EAAE,CAAC;YACb,MAAM,MAAM,GAAG,KAAK,CAAC,OAAO,EAAE,CAAC;YAC/B,IAAI,CAAC,MAAM,EAAE,CAAC;gBACb,MAAM,KAAK,CAAC,WAAW,CAAC,CAAC;gBACzB,SAAS;YACV,CAAC;YAED,GAAG,CAAC,MAAM,EAAE,KAAK,EAAE,CAAC,oBAAoB,EAAE;gBACzC,MAAM,EAAE,WAAW;gBACnB,GAAG,EAAE,MAAM;aACX,CAAC,CAAC;YACH,IAAI,CAAC;gBACJ,MAAM,eAAe,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;YACpC,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBAChB,GAAG,CAAC,MAAM,EAAE,KAAK,EAAE,CAAC,mBAAmB,EAAE;oBACxC,GAAG,EAAE,KAAK;oBACV,GAAG,EAAE,MAAM;oBACX,MAAM,EAAE,WAAW;iBACnB,CAAC,CAAC;YACJ,CAAC;QACF,CAAC;IACF,CAAC;IAED,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,WAAW,EAAE,CAAC,EAAE,EAAE,CAAC;QACtC,UAAU,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IACnB,CAAC;IAED,GAAG,CAAC,MAAM,EAAE,KAAK,EAAE,CAAC,4BAA4B,EAAE,EAAE,WAAW,EAAE,CAAC,CAAC;AACpE,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,2BAA2B,CAAC,IAIjD;IACA,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,sBAAsB,EAAE,CAAC;IAEzD,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACxB,IAAI,CAAC,MAAM,EAAE,KAAK,EAAE,CAAC,6BAA6B,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC;QAC7D,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IACzB,CAAC;IAED,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACrB,IAAI,CAAC,MAAM,EAAE,KAAK,EAAE,CAAC,6BAA6B,EAAE,EAAE,KAAK,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;IAC7E,CAAC;IAED,OAAO,IAAI,CAAC,MAAM,CAAC;AACpB,CAAC"}

package/dist/skills/skill-generator.d.ts

@@ -0,0 +1,16 @@
+import type { FileStorage } from "../storage/file-storage.js";
+import type { MetadataSchemaItem } from "../types.js";
+export interface SkillGeneratorOptions {
+    domain: string;
+    sources: string[];
+    baseDir: string;
+    metadataSchema: MetadataSchemaItem[];
+    overwrite: boolean;
+}
+/**
+ * Generate a Claude Code skill file for searching indexed data
+ */
+export declare function generateSkill(options: SkillGeneratorOptions, fileStorage: FileStorage): Promise<{
+    skillPath: string;
+}>;
+//# sourceMappingURL=skill-generator.d.ts.map

package/dist/skills/skill-generator.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"skill-generator.d.ts","sourceRoot":"","sources":["../../src/skills/skill-generator.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,4BAA4B,CAAC;AAC9D,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,aAAa,CAAC;AAGtD,MAAM,WAAW,qBAAqB;IACrC,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,cAAc,EAAE,kBAAkB,EAAE,CAAC;IACrC,SAAS,EAAE,OAAO,CAAC;CACnB;AAqND;;GAEG;AACH,wBAAsB,aAAa,CAClC,OAAO,EAAE,qBAAqB,EAC9B,WAAW,EAAE,WAAW,GACtB,OAAO,CAAC;IAAE,SAAS,EAAE,MAAM,CAAA;CAAE,CAAC,CAsBhC"}

package/dist/skills/skill-generator.js

@@ -0,0 +1,210 @@
+import { mkdir, writeFile } from "node:fs/promises";
+import path from "node:path";
+import { fileExists } from "../utils/file.js";
+/**
+ * Generate ripgrep example patterns for a metadata field
+ */
+function generateRgPattern(field, sampleValue, isArray) {
+    if (isArray) {
+        // For array fields, search within YAML array syntax
+        return `rg "${field}:\\s*\\[.*${sampleValue}.*\\]" processed`;
+    }
+    // For single-value fields, search for exact value
+    return `rg "${field}:\\s*${sampleValue}" processed`;
+}
+function generateSkillName(sources, maxLength = 30) {
+    let sourcesStr = sources.join("-");
+    sourcesStr = sourcesStr.trim();
+    sourcesStr = sourcesStr.toLowerCase().replace(/[^a-z0-9]+/g, "-");
+    sourcesStr = sourcesStr.replace(/^-+/, "").replace(/-+$/, "");
+    sourcesStr = sourcesStr.replace(/-+/g, "-");
+    let skillName = `search-${sourcesStr}`;
+    if (skillName.length > maxLength) {
+        skillName = skillName.slice(0, maxLength);
+    }
+    return skillName;
+}
+function sanitizePathSegment(name, maxLength = 50) {
+    let sanitized = name
+        .trim()
+        .toLowerCase()
+        .replace(/[^a-z0-9]+/g, "-");
+    sanitized = sanitized.replace(/^-+/, "").replace(/-+$/, "");
+    sanitized = sanitized.replace(/-+/g, "-");
+    if (sanitized.length > maxLength) {
+        sanitized = sanitized.slice(0, maxLength);
+    }
+    return sanitized || "unknown";
+}
+/**
+ * Generate the skill content from a template
+ */
+function generateSkillContent(domain, sources, metadataSchema, fileStorage) {
+    const skillName = generateSkillName(sources);
+    // Generate metadata list from schema
+    const metadataList = metadataSchema
+        .map((field) => {
+        const typeSuffix = field.type.startsWith("enum") && field.enumValues
+            ? ` (values: ${field.enumValues.join(", ")})`
+            : "";
+        return `- \`${field.name}\` - *${field.type}*${typeSuffix}`;
+    })
+        .join("\n");
+    // Generate ripgrep examples for first 3-4 fields, showing both array and single-value patterns
+    const exampleFields = metadataSchema.slice(0, 4);
+    const rgExamples = exampleFields
+        .map((field) => {
+        // Check if field type indicates an array (e.g., "string[]", "enum[]")
+        const typeStr = String(field.type);
+        const isArray = typeStr.includes("[]");
+        const sampleValue = field.enumValues?.[0] ?? "VALUE";
+        const example = generateRgPattern(field.name, sampleValue, isArray);
+        return `# By ${field.name}\n${example}`;
+    })
+        .join("\n\n");
+    const exampleSource = sanitizePathSegment(sources[0] ?? "source");
+    return `---
+name: ${skillName}
+description: Guide for searching and analyzing indexed content from ${sources.join(", ")} in the '${domain}' domain. This skill should be used when you need information from ${sources.join(", ")} sources to answer questions or conduct research.
+---
+
+# Skill Overview
+
+This skill provides guidance for efficient search over indexed content from ${sources.join(", ")} in the \`${domain}\` domain. It leverages grep-friendly metadata and chunked content storage to enable precise filtering and retrieval.
+
+## About the Content
+
+Content from ${sources.join(", ")} has been fetched, chunked, enriched with searchable metadata, and stored in the '${fileStorage.baseDir}' directory.
+
+### Directory Structure
+
+\`\`\`
+${fileStorage.baseDir}/
+├── processed/       # Cleaned, search-optimized content with metadata
+│   └── {source}/    # ${sources.join(", ")}, etc.
+│       └── {publisher?}/
+│           └── YYYY-MM/
+│               └── YYYY-MM-DD-label.md
+└── raw/             # Original raw content as ingested (mirrors processed/)
+\`\`\`
+
+### File Format
+
+Each processed file contains YAML frontmatter with document metadata and a \`chunks\` array. Each chunk includes:
+- \`id\`: Unique chunk identifier (e.g., c01, c02)
+- \`title\`: Chunk title
+- Domain-specific metadata fields
+
+Chunked content follows the frontmatter, with chunk IDs as section headers.
+
+\`\`\`yaml
+---
+title: "Document Title"
+source: "Source Name"
+publisher: "Publisher Name"
+<other metadata fields>
+chunks:
+  - id: c01
+    title: "First Chunk Title"
+    # Domain-specific metadata fields related to this chunk
+  - id: c02
+    title: "Second Chunk Title"
+    # Domain-specific metadata fields related to this chunk
+---
+
+CHUNK c01: "First Chunk Title"
+<chunk content here>
+
+CHUNK c02: "Second Chunk Title"
+<chunk content here>
+\`\`\`
+
+The YAML frontmatter serves as an index for the entire document.
+
+### Key Metadata Fields
+
+Below are the most common metadata fields with sample values. Additional metadata fields and values may exist beyond those listed here.
+
+${metadataList}
+
+## Recommended Search Strategy
+
+0. **ALWAYS use rg (ripgrep)**:  
+   Ripgrep is optimized for searching large codebases and text files quickly. It supports regex, file path patterns, and context capture.
+   It MUST be your primary search tool for this content if installed.
+
+1. **Constrain by time range first**  
+   Use file path patterns (e.g., \`YYYY/MM/YYYY-MM-DD\`) to limit the search space before inspecting content.
+
+2. **Apply metadata filters**  
+   Use \`ripgrep\` to match specific YAML frontmatter fields. Note that metadata fields can be either:
+   - **Single values**: Match with \`field: value\` (e.g., \`date: 2025-01-15\`)
+   - **Arrays**: Match with \`field: [ value1, value2 ]\` or search within arrays using \`field:\\s*\\[.*value.*\\]\`
+   
+   Refer to the Key Metadata Fields section below to understand which fields are arrays vs single values.
+
+3. **Leverage YAML frontmatter as a document index**  
+   Treat frontmatter as a document summary. Read it first to understand:
+   - Which chunks exist
+   - What each chunk covers  
+   This avoids unnecessary full-content reads.
+
+4. **Identify relevant chunks**  
+   From search results and frontmatter, collect IDs of chunks likely to contain relevant information.
+
+5. **Enumerate candidate documents**  
+   Before reading chunk content, broaden queries slightly (alternative wording, synonyms, metadata variations) to ensure all relevant documents and chunk IDs are discovered.
+
+6. **Refine iteratively**  
+   Adjust path patterns, metadata filters, and query terms based on findings until no new relevant documents or chunks appear.
+
+7. **Read targeted content only**  
+   Use collected chunk IDs to read only the necessary sections of each document.
+
+## Ripgrep Search Examples
+
+\`\`\`bash
+# Find specific chunk content with context
+rg "CHUNK c01:" -A 20 processed
+
+# Search only ${exampleSource} content
+rg "search query" processed/${exampleSource}
+
+# Search ${exampleSource} content from December 2025
+rg "search query" processed/${exampleSource} --glob "**/2025-12/*.md"
+
+# Combine multiple metadata filters
+rg -l "field1:.*value1" processed | xargs rg "field2:.*value2"
+
+# List unique values for a metadata field
+rg "field_name:" processed | sort | uniq -c | sort -rn | head -20
+
+${rgExamples}
+\`\`\`
+
+## Important Guidelines
+
+- **Primary source**: Always use the \`processed/\` directory for searches; only fall back to \`raw/\` if necessary
+- **Metadata first**: Start with metadata filtering before full-text content search
+- **Context capture**: Use \`-B\` and \`-A\` flags to capture surrounding lines for context without reading entire chunks
+- **Citation style**: When referencing content, cite by source name, publisher, and title—never expose internal structure like chunk IDs or file paths to the user
+`;
+}
+/**
+ * Generate a Claude Code skill file for searching indexed data
+ */
+export async function generateSkill(options, fileStorage) {
+    const skillName = generateSkillName(options.sources, 30);
+    const skillContent = generateSkillContent(options.domain, options.sources, options.metadataSchema, fileStorage);
+    // Create skill directory
+    const skillDir = path.join(options.baseDir, ".claude", "skills", skillName);
+    await mkdir(skillDir, { recursive: true });
+    // Write skill file
+    const skillPath = path.join(skillDir, "SKILL.md");
+    const skillExists = await fileExists(skillPath);
+    if (!skillExists || options.overwrite) {
+        await writeFile(skillPath, skillContent, "utf8");
+    }
+    return { skillPath };
+}
+//# sourceMappingURL=skill-generator.js.map

package/dist/skills/skill-generator.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"skill-generator.js","sourceRoot":"","sources":["../../src/skills/skill-generator.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AACpD,OAAO,IAAI,MAAM,WAAW,CAAC;AAG7B,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAU9C;;GAEG;AACH,SAAS,iBAAiB,CACzB,KAAa,EACb,WAAmB,EACnB,OAAgB;IAEhB,IAAI,OAAO,EAAE,CAAC;QACb,oDAAoD;QACpD,OAAO,OAAO,KAAK,aAAa,WAAW,kBAAkB,CAAC;IAC/D,CAAC;IACD,kDAAkD;IAClD,OAAO,OAAO,KAAK,QAAQ,WAAW,aAAa,CAAC;AACrD,CAAC;AAED,SAAS,iBAAiB,CAAC,OAAiB,EAAE,SAAS,GAAG,EAAE;IAC3D,IAAI,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACnC,UAAU,GAAG,UAAU,CAAC,IAAI,EAAE,CAAC;IAC/B,UAAU,GAAG,UAAU,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,aAAa,EAAE,GAAG,CAAC,CAAC;IAClE,UAAU,GAAG,UAAU,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;IAC9D,UAAU,GAAG,UAAU,CAAC,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;IAE5C,IAAI,SAAS,GAAG,UAAU,UAAU,EAAE,CAAC;IACvC,IAAI,SAAS,CAAC,MAAM,GAAG,SAAS,EAAE,CAAC;QAClC,SAAS,GAAG,SAAS,CAAC,KAAK,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC;IAC3C,CAAC;IAED,OAAO,SAAS,CAAC;AAClB,CAAC;AAED,SAAS,mBAAmB,CAAC,IAAY,EAAE,SAAS,GAAG,EAAE;IACxD,IAAI,SAAS,GAAG,IAAI;SAClB,IAAI,EAAE;SACN,WAAW,EAAE;SACb,OAAO,CAAC,aAAa,EAAE,GAAG,CAAC,CAAC;IAC9B,SAAS,GAAG,SAAS,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;IAC5D,SAAS,GAAG,SAAS,CAAC,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;IAE1C,IAAI,SAAS,CAAC,MAAM,GAAG,SAAS,EAAE,CAAC;QAClC,SAAS,GAAG,SAAS,CAAC,KAAK,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC;IAC3C,CAAC;IAED,OAAO,SAAS,IAAI,SAAS,CAAC;AAC/B,CAAC;AAED;;GAEG;AACH,SAAS,oBAAoB,CAC5B,MAAc,EACd,OAAiB,EACjB,cAAoC,EACpC,WAAwB;IAExB,MAAM,SAAS,GAAG,iBAAiB,CAAC,OAAO,CAAC,CAAC;IAE7C,qCAAqC;IACrC,MAAM,YAAY,GAAG,cAAc;SACjC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE;QACd,MAAM,UAAU,GACf,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,UAAU;YAChD,CAAC,CAAC,aAAa,KAAK,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG;YAC7C,CAAC,CAAC,EAAE,CAAC;QACP,OAAO,OAAO,KAAK,CAAC,IAAI,SAAS,KAAK,CAAC,IAAI,IAAI,UAAU,EAAE,CAAC;IAC7D,CAAC,CAAC;SACD,IAAI,CAAC,IAAI,CAAC,CAAC;IAEb,+FAA+F;IAC/F,MAAM,aAAa,GAAG,cAAc,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IACjD,MAAM,UAAU,GAAG,aAAa;SAC9B,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE;QACd,sEAAsE;QACtE,MAAM,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QACnC,MAAM,OAAO,GAAG,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;QACvC,MAAM,WAAW,GAAG,KAAK,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC,IAAI,OAAO,CAAC;QACrD,MAAM,OAAO,GAAG,iBAAiB,CAAC,KAAK,CAAC,IAAI,EAAE,WAAW,EAAE,OAAO,CAAC,CAAC;QACpE,OAAO,QAAQ,KAAK,CAAC,IAAI,KAAK,OAAO,EAAE,CAAC;IACzC,CAAC,CAAC;SACD,IAAI,CAAC,MAAM,CAAC,CAAC;IAEf,MAAM,aAAa,GAAG,mBAAmB,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,QAAQ,CAAC,CAAC;IAElE,OAAO;QACA,SAAS;sEACqD,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,YAAY,MAAM,sEAAsE,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC;;;;;8EAKpH,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,aAAa,MAAM;;;;eAIpG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,qFAAqF,WAAW,CAAC,OAAO;;;;;EAKvI,WAAW,CAAC,OAAO;;yBAEI,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EA4CzC,YAAY;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;gBA0CE,aAAa;8BACC,aAAa;;WAEhC,aAAa;8BACM,aAAa;;;;;;;;EAQzC,UAAU;;;;;;;;;CASX,CAAC;AACF,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,aAAa,CAClC,OAA8B,EAC9B,WAAwB;IAExB,MAAM,SAAS,GAAG,iBAAiB,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;IACzD,MAAM,YAAY,GAAG,oBAAoB,CACxC,OAAO,CAAC,MAAM,EACd,OAAO,CAAC,OAAO,EACf,OAAO,CAAC,cAAc,EACtB,WAAW,CACX,CAAC;IAEF,yBAAyB;IACzB,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,SAAS,EAAE,QAAQ,EAAE,SAAS,CAAC,CAAC;IAC5E,MAAM,KAAK,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAE3C,mBAAmB;IACnB,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;IAClD,MAAM,WAAW,GAAG,MAAM,UAAU,CAAC,SAAS,CAAC,CAAC;IAEhD,IAAI,CAAC,WAAW,IAAI,OAAO,CAAC,SAAS,EAAE,CAAC;QACvC,MAAM,SAAS,CAAC,SAAS,EAAE,YAAY,EAAE,MAAM,CAAC,CAAC;IAClD,CAAC;IAED,OAAO,EAAE,SAAS,EAAE,CAAC;AACtB,CAAC"}

package/dist/storage/file-storage.d.ts

@@ -0,0 +1,16 @@
+import type { DocumentRef, DocumentAddResult as DocumentSaveResult } from "../storage/types.js";
+import type { GreptorEatInput, Metadata } from "../types.js";
+export interface FileStorage {
+    readonly baseDir: string;
+    readonly rawContentPath: string;
+    readonly processedContentPath: string;
+    saveRawContent(input: GreptorEatInput): Promise<DocumentSaveResult>;
+    readRawContent(ref: DocumentRef): Promise<{
+        metadata: Metadata;
+        content: string;
+    }>;
+    getUnprocessedContents(): Promise<DocumentRef[]>;
+    saveProcessedContent(ref: DocumentRef, content: string): Promise<void>;
+}
+export declare function createFileStorage(baseDir: string): FileStorage;
+//# sourceMappingURL=file-storage.d.ts.map

package/dist/storage/file-storage.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"file-storage.d.ts","sourceRoot":"","sources":["../../src/storage/file-storage.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EACX,WAAW,EACX,iBAAiB,IAAI,kBAAkB,EACvC,MAAM,qBAAqB,CAAC;AAC7B,OAAO,KAAK,EAAE,eAAe,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAG7D,MAAM,WAAW,WAAW;IAC3B,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,QAAQ,CAAC,oBAAoB,EAAE,MAAM,CAAC;IAEtC,cAAc,CAAC,KAAK,EAAE,eAAe,GAAG,OAAO,CAAC,kBAAkB,CAAC,CAAC;IACpE,cAAc,CACb,GAAG,EAAE,WAAW,GACd,OAAO,CAAC;QAAE,QAAQ,EAAE,QAAQ,CAAC;QAAC,OAAO,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IACpD,sBAAsB,IAAI,OAAO,CAAC,WAAW,EAAE,CAAC,CAAC;IACjD,oBAAoB,CAAC,GAAG,EAAE,WAAW,EAAE,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CACvE;AAED,wBAAgB,iBAAiB,CAAC,OAAO,EAAE,MAAM,GAAG,WAAW,CAyN9D"}