onadit 0.0.1 → 0.1.0-beta.45

package/dist/src/ingestion/scanner.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"scanner.js","sourceRoot":"","sources":["../../../src/ingestion/scanner.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AACH,OAAO,EAAE,MAAM,SAAS,CAAC;AACzB,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAC7C,OAAO,EAAE,MAAM,EAAkB,MAAM,cAAc,CAAC;AACtD,OAAO,EAAE,QAAQ,EAAE,MAAM,sBAAsB,CAAC;AAEhD,OAAO,EAAE,WAAW,EAAE,MAAM,sBAAsB,CAAC;AACnD,OAAO,EAAE,oBAAoB,EAAE,MAAM,cAAc,CAAC;AAoBpD,MAAM,OAAO,OAAO;IACV,MAAM,GAAG,KAAK,CAAC;IACN,SAAS,CAAS;IAC3B,KAAK,GAAc,EAAE,YAAY,EAAE,IAAI,EAAE,YAAY,EAAE,IAAI,EAAE,CAAC;IAEtE,YAAY,OAAe;QACzB,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,iBAAiB,CAAC,CAAC;QACvD,IAAI,CAAC,SAAS,EAAE,CAAC;IACnB,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,IAAI,CACR,IAAuB,EACvB,OAAoB,EAAE;QAEtB,IAAI,IAAI,CAAC,MAAM;YAAE,OAAO,EAAE,MAAM,EAAE,EAAE,EAAE,YAAY,EAAE,CAAC,EAAE,CAAC;QAExD,IAAI,YAAY,GAAG,CAAC,CAAC;QACrB,MAAM,MAAM,GAAgB,EAAE,CAAC;QAC/B,MAAM,KAAK,GAAG,QAAQ,CAAC,iBAAiB,CAAC,CAAC;QAE1C,MAAM,QAAQ,GAAgB,IAAI,CAAC,YAAY;YAC7C,CAAC,CAAC,EAAE,YAAY,EAAE,IAAI,CAAC,YAAY,EAAE;YACrC,CAAC,CAAC,EAAE,CAAC;QAEP,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;YACvB,MAAM,KAAK,GAAG,aAAa,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;YAC3C,KAAK,MAAM,QAAQ,IAAI,KAAK,EAAE,CAAC;gBAC7B,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC;oBAAE,SAAS;gBACrC,YAAY,EAAE,CAAC;gBAEf,MAAM,EAAE,GAAG,MAAM,CAAC,QAAQ,CAAC,CAAC;gBAC5B,MAAM,QAAQ,GAAG,CAAC,MAAM,KAAK;qBAC1B,KAAK,EAAE;qBACP,KAAK,CAAC,SAAS,EAAE,GAAG,CAAC;qBACrB,OAAO,EAAE,CAAuB,CAAC;gBAEpC,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;oBAC1B,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC,CAAC;gBAC/C,CAAC;qBAAM,CAAC;oBACN,MAAM,KAAK,GAAG,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;oBACpC,MAAM,SAAS,GAAG,IAAI,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAE,CAAC,UAAU,CAAC,CAAC;oBACpD,IAAI,KAAK,CAAC,KAAK,GAAG,SAAS,EAAE,CAAC;wBAC5B,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC,CAAC;oBAClD,CAAC;gBACH,CAAC;YACH,CAAC;QACH,CAAC;QAED,IAAI,CAAC,KAAK,CAAC,YAAY,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;QACnD,MAAM,IAAI,CAAC,SAAS,EAAE,CAAC;QAEvB,OAAO,EAAE,MAAM,EAAE,YAAY,EAAE,CAAC;IAClC,CAAC;IAED,KAAK;QACH,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC;IACrB,CAAC;IAED,MAAM;QACJ,IAAI,CAAC,MAAM,GAAG,KAAK,CAAC;IACtB,CAAC;IAED,IAAI,QAAQ;QACV,OAAO,IAAI,CAAC,MAAM,CAAC;IACrB,CAAC;IAED,IAAI,SAAS;QACX,OAAO,EAAE,GAAG,IAAI,CAAC,KAAK,EAAE,CAAC;IAC3B,CAAC;IAED,WAAW,CAAC,UAAkB;QAC5B,IAAI,CAAC,KAAK,CAAC,YAAY,GAAG,UAAU,CAAC;QACrC,IAAI,CAAC,SAAS,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;IACnC,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,eAAuB;QAC3B,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,YAAY;YAAE,OAAO,IAAI,CAAC;QAC1C,MAAM,QAAQ,GAAG,IAAI,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC,OAAO,EAAE,CAAC;QAC7D,OAAO,IAAI,CAAC,GAAG,EAAE,IAAI,QAAQ,GAAG,eAAe,GAAG,IAAI,CAAC;IACzD,CAAC;IAEO,SAAS;QACf,IAAI,CAAC;YACH,MAAM,GAAG,GAAG,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;YACrD,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAc,CAAC;QAC5C,CAAC;QAAC,MAAM,CAAC;YACP,mCAAmC;QACrC,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,SAAS;QACrB,MAAM,WAAW,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;IACzE,CAAC;CACF;AAED;;;;;;GAMG;AACH,MAAM,UAAU,aAAa,CAAC,GAAW,EAAE,OAAoB,EAAE;IAC/D,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC;QAAE,OAAO,OAAO,CAAC;IACxC,MAAM,YAAY,GAAG,IAAI,CAAC,YAAY,IAAI,EAAE,CAAC;IAE7C,MAAM,OAAO,GAAG,EAAE,CAAC,WAAW,CAAC,GAAG,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC;IAC7D,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;QAC5B,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;QAC5C,IAAI,oBAAoB,CAAC,QAAQ,EAAE,YAAY,CAAC;YAAE,SAAS;QAC3D,IAAI,KAAK,CAAC,WAAW,EAAE,IAAI,CAAC,KAAK,CAAC,cAAc,EAAE,EAAE,CAAC;YACnD,OAAO,CAAC,IAAI,CAAC,GAAG,aAAa,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC,CAAC;QACjD,CAAC;aAAM,IAAI,KAAK,CAAC,MAAM,EAAE,EAAE,CAAC;YAC1B,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QACzB,CAAC;IACH,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC"}

package/dist/src/ingestion/skip-quarantine.d.ts

@@ -0,0 +1,10 @@
+export type SkipReason = "empty_extracted_content" | "inbox_unsupported_extension";
+export interface SkipQuarantineRecord {
+    kind: "indexer_skip";
+    reason: SkipReason;
+    source_path: string;
+    metadata?: Record<string, unknown>;
+    quarantined_at: string;
+}
+export declare function writeSkipQuarantine(dataDir: string, record: Omit<SkipQuarantineRecord, "kind" | "quarantined_at">): Promise<string>;
+//# sourceMappingURL=skip-quarantine.d.ts.map

package/dist/src/ingestion/skip-quarantine.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"skip-quarantine.d.ts","sourceRoot":"","sources":["../../../src/ingestion/skip-quarantine.ts"],"names":[],"mappings":"AAiBA,MAAM,MAAM,UAAU,GAClB,yBAAyB,GACzB,6BAA6B,CAAC;AAElC,MAAM,WAAW,oBAAoB;IACnC,IAAI,EAAE,cAAc,CAAC;IACrB,MAAM,EAAE,UAAU,CAAC;IACnB,WAAW,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACnC,cAAc,EAAE,MAAM,CAAC;CACxB;AAED,wBAAsB,mBAAmB,CACvC,OAAO,EAAE,MAAM,EACf,MAAM,EAAE,IAAI,CAAC,oBAAoB,EAAE,MAAM,GAAG,gBAAgB,CAAC,GAC5D,OAAO,CAAC,MAAM,CAAC,CAwBjB"}

package/dist/src/ingestion/skip-quarantine.js

@@ -0,0 +1,35 @@
+/**
+ * Quarantine record writer for the watcher/indexer path.
+ *
+ * When a watched file cannot be safely indexed (e.g. extraction yielded
+ * empty content, which would cause a Voyage 400), we record a structured
+ * skip event to `data/quarantine/` so the user can see why a file was
+ * dropped — instead of letting it surface only as a stderr line.
+ *
+ * Distinct from the conversation-level quarantine in `jobs/handlers/ingest.ts`
+ * (that one carries `conversation_id` + `content_preview`); this one carries
+ * `source_path` + `reason` + extractor metadata, which is what the watcher
+ * has on hand.
+ */
+import crypto from "node:crypto";
+import path from "node:path";
+import { atomicWrite } from "../storage/atomic.js";
+export async function writeSkipQuarantine(dataDir, record) {
+    const timestamp = new Date().toISOString().replace(/[:.]/g, "-");
+    const hash = crypto
+        .createHash("sha256")
+        .update(record.source_path)
+        .digest("hex")
+        .slice(0, 12);
+    const file = path.join(dataDir, "quarantine", `indexer-skip_${record.reason}_${timestamp}_${hash}.json`);
+    const payload = {
+        kind: "indexer_skip",
+        reason: record.reason,
+        source_path: record.source_path,
+        ...(record.metadata ? { metadata: record.metadata } : {}),
+        quarantined_at: new Date().toISOString(),
+    };
+    await atomicWrite(file, JSON.stringify(payload, null, 2));
+    return file;
+}
+//# sourceMappingURL=skip-quarantine.js.map

package/dist/src/ingestion/skip-quarantine.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"skip-quarantine.js","sourceRoot":"","sources":["../../../src/ingestion/skip-quarantine.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AACH,OAAO,MAAM,MAAM,aAAa,CAAC;AACjC,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,WAAW,EAAE,MAAM,sBAAsB,CAAC;AAcnD,MAAM,CAAC,KAAK,UAAU,mBAAmB,CACvC,OAAe,EACf,MAA6D;IAE7D,MAAM,SAAS,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;IACjE,MAAM,IAAI,GAAG,MAAM;SAChB,UAAU,CAAC,QAAQ,CAAC;SACpB,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC;SAC1B,MAAM,CAAC,KAAK,CAAC;SACb,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IAEhB,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CACpB,OAAO,EACP,YAAY,EACZ,gBAAgB,MAAM,CAAC,MAAM,IAAI,SAAS,IAAI,IAAI,OAAO,CAC1D,CAAC;IAEF,MAAM,OAAO,GAAyB;QACpC,IAAI,EAAE,cAAc;QACpB,MAAM,EAAE,MAAM,CAAC,MAAM;QACrB,WAAW,EAAE,MAAM,CAAC,WAAW;QAC/B,GAAG,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACzD,cAAc,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;KACzC,CAAC;IAEF,MAAM,WAAW,CAAC,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;IAC1D,OAAO,IAAI,CAAC;AACd,CAAC"}

package/dist/src/ingestion/staging.d.ts

@@ -0,0 +1,109 @@
+import { type IngestSource } from "./parsers/types.js";
+/** Files smaller than this are classified as `too_small`. */
+export declare const MIN_USEFUL_BYTES = 512;
+/** Per-source extension allowlist for preflight classification. */
+export declare const SOURCE_EXTENSIONS: Record<IngestSource, readonly string[]>;
+/**
+ * Junk markers — see module header for meanings. Stable strings; persisted
+ * into preflight.json and inspected by tests.
+ */
+export type JunkMarker = "empty" | "too_small" | "non_utf8" | "duplicate_hash" | "no_assistant_turns" | "unsupported_format";
+export interface ManifestFile {
+    name: string;
+    path: string;
+    size_bytes: number;
+}
+export interface Manifest {
+    batch_name: string;
+    created_at: string;
+    source: IngestSource;
+    path: string;
+    files: ManifestFile[];
+}
+export interface PreflightFile {
+    name: string;
+    path: string;
+    size_bytes: number;
+    extension: string;
+    supported: boolean;
+    utf8_ok: boolean;
+    content_hash: string | null;
+    duplicate_hash: boolean;
+    junk_markers: JunkMarker[];
+    notes: string[];
+}
+export interface SizeDistribution {
+    min: number;
+    max: number;
+    median: number;
+    total: number;
+}
+export interface Preflight {
+    batch_name: string;
+    source: IngestSource;
+    generated_at: string;
+    total_files: number;
+    supported_files: number;
+    unsupported_files: number;
+    size_distribution: SizeDistribution;
+    duplicate_candidates: number;
+    parseability: {
+        parseable: number;
+        unparseable: number;
+    };
+    junk_candidates: number;
+    files: PreflightFile[];
+}
+export interface StagingArtifacts {
+    manifest: Manifest;
+    preflight: Preflight;
+}
+/**
+ * Cheap-sample size for utf-8 / zip-magic checks on files that are too
+ * large to fully validate. Files at or below this size are read in full
+ * by the same code path (sample === whole file).
+ */
+export declare const PARSE_SAMPLE_BYTES: number;
+/**
+ * Largest file we will fully read into memory for structural JSON
+ * validation. Above this cap, JSON files are flagged as
+ * `unvalidated_too_large` instead of being trusted on a sniff. The cap
+ * comfortably accommodates real ChatGPT/Claude exports while protecting
+ * against pathological inputs.
+ */
+export declare const FULL_VALIDATION_CAP_BYTES: number;
+/**
+ * List every file in a batch folder, excluding our own manifest/preflight
+ * artifacts. Returns absolute paths, sorted by name for deterministic output.
+ */
+export declare function listBatchFiles(batchDir: string): string[];
+/**
+ * Build a manifest for a staged batch. Does not touch file contents.
+ */
+export declare function buildManifest(params: {
+    batchDir: string;
+    source: IngestSource;
+    now?: Date;
+}): Manifest;
+/**
+ * Run preflight classification on every file listed in the manifest.
+ * No network, no model calls. Reads up to PARSE_SAMPLE_BYTES per file.
+ */
+export declare function runPreflight(params: {
+    manifest: Manifest;
+    now?: Date;
+}): Preflight;
+/**
+ * Persist manifest and preflight as `manifest.json` and `preflight.json`
+ * inside the batch directory. Overwrites any previous artifacts.
+ */
+export declare function writeStagingArtifacts(batchDir: string, artifacts: StagingArtifacts): void;
+/**
+ * Convenience wrapper: build manifest, run preflight, persist both.
+ */
+export declare function stageBatch(params: {
+    batchDir: string;
+    source: IngestSource;
+    now?: Date;
+}): StagingArtifacts;
+//# sourceMappingURL=staging.d.ts.map

package/dist/src/ingestion/staging.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"staging.d.ts","sourceRoot":"","sources":["../../../src/ingestion/staging.ts"],"names":[],"mappings":"AAyBA,OAAO,EAAkB,KAAK,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAEvE,6DAA6D;AAC7D,eAAO,MAAM,gBAAgB,MAAM,CAAC;AAEpC,mEAAmE;AACnE,eAAO,MAAM,iBAAiB,EAAE,MAAM,CAAC,YAAY,EAAE,SAAS,MAAM,EAAE,CAKrE,CAAC;AAEF;;;GAGG;AACH,MAAM,MAAM,UAAU,GAClB,OAAO,GACP,WAAW,GACX,UAAU,GACV,gBAAgB,GAChB,oBAAoB,GACpB,oBAAoB,CAAC;AAEzB,MAAM,WAAW,YAAY;IAC3B,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,QAAQ;IACvB,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,YAAY,CAAC;IACrB,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,YAAY,EAAE,CAAC;CACvB;AAED,MAAM,WAAW,aAAa;IAC5B,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,EAAE,MAAM,CAAC;IACnB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,OAAO,CAAC;IACnB,OAAO,EAAE,OAAO,CAAC;IACjB,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,cAAc,EAAE,OAAO,CAAC;IACxB,YAAY,EAAE,UAAU,EAAE,CAAC;IAC3B,KAAK,EAAE,MAAM,EAAE,CAAC;CACjB;AAED,MAAM,WAAW,gBAAgB;IAC/B,GAAG,EAAE,MAAM,CAAC;IACZ,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;CACf;AAED,MAAM,WAAW,SAAS;IACxB,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,YAAY,CAAC;IACrB,YAAY,EAAE,MAAM,CAAC;IACrB,WAAW,EAAE,MAAM,CAAC;IACpB,eAAe,EAAE,MAAM,CAAC;IACxB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,iBAAiB,EAAE,gBAAgB,CAAC;IACpC,oBAAoB,EAAE,MAAM,CAAC;IAC7B,YAAY,EAAE;QACZ,SAAS,EAAE,MAAM,CAAC;QAClB,WAAW,EAAE,MAAM,CAAC;KACrB,CAAC;IACF,eAAe,EAAE,MAAM,CAAC;IACxB,KAAK,EAAE,aAAa,EAAE,CAAC;CACxB;AAED,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,EAAE,QAAQ,CAAC;IACnB,SAAS,EAAE,SAAS,CAAC;CACtB;AAeD;;;;GAIG;AACH,eAAO,MAAM,kBAAkB,QAAc,CAAC;AAE9C;;;;;;GAMG;AACH,eAAO,MAAM,yBAAyB,QAAoB,CAAC;AAO3D;;;GAGG;AACH,wBAAgB,cAAc,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,EAAE,CAMzD;AAED;;GAEG;AACH,wBAAgB,aAAa,CAAC,MAAM,EAAE;IACpC,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,YAAY,CAAC;IACrB,GAAG,CAAC,EAAE,IAAI,CAAC;CACZ,GAAG,QAAQ,CAoBX;AAED;;;GAGG;AACH,wBAAgB,YAAY,CAAC,MAAM,EAAE;IACnC,QAAQ,EAAE,QAAQ,CAAC;IACnB,GAAG,CAAC,EAAE,IAAI,CAAC;CACZ,GAAG,SAAS,CAkDZ;AAED;;;GAGG;AACH,wBAAgB,qBAAqB,CAAC,QAAQ,EAAE,MAAM,EAAE,SAAS,EAAE,gBAAgB,GAAG,IAAI,CAWzF;AAED;;GAEG;AACH,wBAAgB,UAAU,CAAC,MAAM,EAAE;IACjC,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,YAAY,CAAC;IACrB,GAAG,CAAC,EAAE,IAAI,CAAC;CACZ,GAAG,gBAAgB,CAOnB"}

package/dist/src/ingestion/staging.js

@@ -0,0 +1,411 @@
+/**
+ * Staged batch preflight — Slice 4.
+ *
+ * A staged batch is a folder on disk containing files the operator intends
+ * to import through the existing `backfill` flow. Before ingestion, the
+ * operator runs a preflight pass that:
+ *   - catalogs every file (manifest.json)
+ *   - classifies each file against junk markers (preflight.json)
+ *
+ * Preflight is cheap on purpose: no model calls, no embeddings, no full
+ * parse of large archives. It reads just enough of each file to detect
+ * obvious problems before the batch poisons the searchable layer.
+ *
+ * Locked junk markers (Slice 4):
+ *   - empty                  → size == 0
+ *   - too_small              → size < MIN_USEFUL_BYTES
+ *   - non_utf8               → bytes do not decode as strict UTF-8
+ *   - duplicate_hash         → same content hash as another file in the batch
+ *   - no_assistant_turns     → source requires assistant turns but none found
+ *   - unsupported_format     → extension not in the per-source allowlist
+ */
+import fs from "node:fs";
+import path from "node:path";
+import crypto from "node:crypto";
+import { INGEST_SOURCES } from "./parsers/types.js";
+/** Files smaller than this are classified as `too_small`. */
+export const MIN_USEFUL_BYTES = 512;
+/** Per-source extension allowlist for preflight classification. */
+export const SOURCE_EXTENSIONS = {
+    chatgpt_web: [".json", ".zip"],
+    claude_web: [".json", ".zip"],
+    grok_web: [".json", ".zip"],
+    gemini_web: [".json"],
+};
+const MANIFEST_FILE = "manifest.json";
+const PREFLIGHT_FILE = "preflight.json";
+const TRIAGE_FILE = "triage.json";
+const CAMPAIGN_REPORT_FILE = "campaign-report.json";
+/** Files written by staging itself — always skipped when listing batch files. */
+const STAGING_ARTIFACT_FILES = new Set([
+    MANIFEST_FILE,
+    PREFLIGHT_FILE,
+    TRIAGE_FILE,
+    CAMPAIGN_REPORT_FILE,
+]);
+/**
+ * Cheap-sample size for utf-8 / zip-magic checks on files that are too
+ * large to fully validate. Files at or below this size are read in full
+ * by the same code path (sample === whole file).
+ */
+export const PARSE_SAMPLE_BYTES = 1024 * 1024;
+/**
+ * Largest file we will fully read into memory for structural JSON
+ * validation. Above this cap, JSON files are flagged as
+ * `unvalidated_too_large` instead of being trusted on a sniff. The cap
+ * comfortably accommodates real ChatGPT/Claude exports while protecting
+ * against pathological inputs.
+ */
+export const FULL_VALIDATION_CAP_BYTES = 256 * 1024 * 1024;
+/** ZIP magic bytes: PK\x03\x04. */
+const ZIP_MAGIC = Buffer.from([0x50, 0x4b, 0x03, 0x04]);
+// --- Public API ---
+/**
+ * List every file in a batch folder, excluding our own manifest/preflight
+ * artifacts. Returns absolute paths, sorted by name for deterministic output.
+ */
+export function listBatchFiles(batchDir) {
+    const entries = fs.readdirSync(batchDir, { withFileTypes: true });
+    return entries
+        .filter((e) => e.isFile() && !STAGING_ARTIFACT_FILES.has(e.name))
+        .map((e) => path.join(batchDir, e.name))
+        .sort();
+}
+/**
+ * Build a manifest for a staged batch. Does not touch file contents.
+ */
+export function buildManifest(params) {
+    const { batchDir } = params;
+    if (!INGEST_SOURCES.includes(params.source)) {
+        throw new Error(`Unknown source: ${params.source}`);
+    }
+    const files = listBatchFiles(batchDir).map((p) => {
+        const stat = fs.statSync(p);
+        return {
+            name: path.basename(p),
+            path: p,
+            size_bytes: stat.size,
+        };
+    });
+    return {
+        batch_name: path.basename(batchDir),
+        created_at: (params.now ?? new Date()).toISOString(),
+        source: params.source,
+        path: path.resolve(batchDir),
+        files,
+    };
+}
+/**
+ * Run preflight classification on every file listed in the manifest.
+ * No network, no model calls. Reads up to PARSE_SAMPLE_BYTES per file.
+ */
+export function runPreflight(params) {
+    const { manifest } = params;
+    const allowed = new Set(SOURCE_EXTENSIONS[manifest.source]);
+    // First pass: per-file classification (without duplicate flag yet).
+    const interim = manifest.files.map((f) => classifyFile(f, manifest.source, allowed));
+    // Second pass: duplicate-hash detection within the batch.
+    const hashCounts = new Map();
+    for (const f of interim) {
+        if (f.content_hash) {
+            hashCounts.set(f.content_hash, (hashCounts.get(f.content_hash) ?? 0) + 1);
+        }
+    }
+    for (const f of interim) {
+        if (f.content_hash && (hashCounts.get(f.content_hash) ?? 0) > 1) {
+            f.duplicate_hash = true;
+            if (!f.junk_markers.includes("duplicate_hash")) {
+                f.junk_markers.push("duplicate_hash");
+            }
+        }
+    }
+    const sizes = interim.map((f) => f.size_bytes);
+    const supportedCount = interim.filter((f) => f.supported).length;
+    // Parseable = supported files that survived every cheap check: size
+    // above the useful floor, valid UTF-8 (or a recognised binary like zip),
+    // readable from disk, and structurally sound (JSON.parse success / zip
+    // magic match). Unsupported/empty/too_small/non_utf8/unreadable/
+    // malformed files never count as parseable.
+    const parseable = interim.filter((f) => isParseable(f)).length;
+    const duplicateCandidates = interim.filter((f) => f.duplicate_hash).length;
+    const junkCandidates = interim.filter((f) => f.junk_markers.length > 0).length;
+    return {
+        batch_name: manifest.batch_name,
+        source: manifest.source,
+        generated_at: (params.now ?? new Date()).toISOString(),
+        total_files: interim.length,
+        supported_files: supportedCount,
+        unsupported_files: interim.length - supportedCount,
+        size_distribution: summarizeSizes(sizes),
+        duplicate_candidates: duplicateCandidates,
+        parseability: {
+            parseable,
+            unparseable: interim.length - parseable,
+        },
+        junk_candidates: junkCandidates,
+        files: interim,
+    };
+}
+/**
+ * Persist manifest and preflight as `manifest.json` and `preflight.json`
+ * inside the batch directory. Overwrites any previous artifacts.
+ */
+export function writeStagingArtifacts(batchDir, artifacts) {
+    fs.writeFileSync(path.join(batchDir, MANIFEST_FILE), JSON.stringify(artifacts.manifest, null, 2) + "\n", "utf-8");
+    fs.writeFileSync(path.join(batchDir, PREFLIGHT_FILE), JSON.stringify(artifacts.preflight, null, 2) + "\n", "utf-8");
+}
+/**
+ * Convenience wrapper: build manifest, run preflight, persist both.
+ */
+export function stageBatch(params) {
+    const manifest = buildManifest(params);
+    const preflight = params.now
+        ? runPreflight({ manifest, now: params.now })
+        : runPreflight({ manifest });
+    writeStagingArtifacts(params.batchDir, { manifest, preflight });
+    return { manifest, preflight };
+}
+// --- Internal helpers ---
+function classifyFile(file, source, allowedExts) {
+    const ext = path.extname(file.name).toLowerCase();
+    const supported = allowedExts.has(ext);
+    const markers = [];
+    const notes = [];
+    if (!supported)
+        markers.push("unsupported_format");
+    if (file.size_bytes === 0)
+        markers.push("empty");
+    else if (file.size_bytes < MIN_USEFUL_BYTES)
+        markers.push("too_small");
+    // Read either the whole file (for structural JSON validation) or just
+    // a sample (for cheap checks on files too large to fully validate).
+    // The same buffer is reused for utf-8 + zip-magic + JSON.parse paths.
+    const willFullyValidate = file.size_bytes > 0 && file.size_bytes <= FULL_VALIDATION_CAP_BYTES;
+    const targetReadBytes = willFullyValidate
+        ? file.size_bytes
+        : Math.min(PARSE_SAMPLE_BYTES, Math.max(file.size_bytes, 0));
+    let sample = null;
+    try {
+        const fd = fs.openSync(file.path, "r");
+        try {
+            const buf = Buffer.alloc(targetReadBytes);
+            const read = buf.length > 0 ? fs.readSync(fd, buf, 0, buf.length, 0) : 0;
+            sample = buf.subarray(0, read);
+        }
+        finally {
+            fs.closeSync(fd);
+        }
+    }
+    catch {
+        notes.push("unreadable");
+    }
+    // Content hash: hash the whole file so duplicate detection is exact.
+    // For huge files this streams; for typical export JSONs it's cheap.
+    let contentHash = null;
+    if (sample) {
+        try {
+            contentHash = hashFile(file.path);
+        }
+        catch {
+            notes.push("hash_failed");
+        }
+    }
+    // UTF-8 check on the sample. ZIPs are binary by design — skip for them.
+    let utf8Ok = true;
+    const isZip = ext === ".zip" || (sample !== null && looksLikeZip(sample));
+    if (sample && !isZip) {
+        utf8Ok = isValidUtf8(sample);
+        if (!utf8Ok)
+            markers.push("non_utf8");
+    }
+    // Parseability:
+    //  - JSON files at or below FULL_VALIDATION_CAP_BYTES are run through
+    //    real JSON.parse. Malformed bytes (even if they start with `[`/`{`
+    //    or contain an assistant-marker substring) are flagged unparseable.
+    //  - JSON files above the cap are flagged `unvalidated_too_large`
+    //    explicitly, so they cannot pass as clean on a sniff and never
+    //    inflate `parseability.parseable`.
+    //  - Zips are validated by PK magic bytes (cheap and conclusive).
+    const fullyRead = sample !== null && sample.length === file.size_bytes;
+    if (sample && supported && file.size_bytes > 0) {
+        if (ext === ".json") {
+            if (fullyRead) {
+                let parsed;
+                let parseOk = false;
+                try {
+                    parsed = JSON.parse(sample.toString("utf-8"));
+                    parseOk = true;
+                }
+                catch {
+                    notes.push("unparseable");
+                }
+                if (parseOk &&
+                    requiresAssistantTurns(source) &&
+                    !hasAssistantTurn(parsed, source)) {
+                    markers.push("no_assistant_turns");
+                }
+            }
+            else {
+                notes.push("unvalidated_too_large");
+            }
+        }
+        else if (ext === ".zip") {
+            if (!looksLikeZip(sample)) {
+                notes.push("unparseable");
+            }
+        }
+    }
+    return {
+        name: file.name,
+        path: file.path,
+        size_bytes: file.size_bytes,
+        extension: ext,
+        supported,
+        utf8_ok: utf8Ok,
+        content_hash: contentHash,
+        duplicate_hash: false,
+        junk_markers: markers,
+        notes,
+    };
+}
+function isParseable(f) {
+    if (!f.supported)
+        return false;
+    if (f.size_bytes < MIN_USEFUL_BYTES)
+        return false;
+    if (!f.utf8_ok)
+        return false;
+    if (f.notes.includes("unparseable"))
+        return false;
+    if (f.notes.includes("unreadable"))
+        return false;
+    // Files too large to fully validate are explicitly NOT parseable —
+    // we have not confirmed structural soundness, so they do not count.
+    if (f.notes.includes("unvalidated_too_large"))
+        return false;
+    return true;
+}
+function hashFile(p) {
+    const hash = crypto.createHash("sha256");
+    const fd = fs.openSync(p, "r");
+    try {
+        const buf = Buffer.alloc(64 * 1024);
+        let read = 0;
+        while ((read = fs.readSync(fd, buf, 0, buf.length, null)) > 0) {
+            hash.update(buf.subarray(0, read));
+        }
+    }
+    finally {
+        fs.closeSync(fd);
+    }
+    return hash.digest("hex");
+}
+function isValidUtf8(buf) {
+    try {
+        new TextDecoder("utf-8", { fatal: true }).decode(buf);
+        return true;
+    }
+    catch {
+        return false;
+    }
+}
+function looksLikeZip(buf) {
+    return buf.length >= ZIP_MAGIC.length && buf.subarray(0, ZIP_MAGIC.length).equals(ZIP_MAGIC);
+}
+/**
+ * Sources whose JSON exports are only meaningful when they contain at
+ * least one assistant turn. ZIPs are excluded from this check (the sample
+ * is compressed binary).
+ */
+function requiresAssistantTurns(source) {
+    return (source === "chatgpt_web" ||
+        source === "claude_web" ||
+        source === "gemini_web" ||
+        source === "grok_web");
+}
+/**
+ * Structural assistant-turn check. Runs only on a successfully parsed
+ * JSON value whose whole content fit in the sample window, so a negative
+ * result is meaningful (not a sampling artifact) and cannot be faked by
+ * a substring that merely looks like an assistant marker.
+ */
+function hasAssistantTurn(parsed, source) {
+    if (source === "chatgpt_web") {
+        if (!Array.isArray(parsed))
+            return false;
+        for (const conv of parsed) {
+            const mapping = conv?.mapping;
+            if (!mapping || typeof mapping !== "object")
+                continue;
+            for (const node of Object.values(mapping)) {
+                const role = node
+                    ?.message?.author?.role;
+                if (role === "assistant")
+                    return true;
+            }
+        }
+        return false;
+    }
+    if (source === "claude_web") {
+        if (!Array.isArray(parsed))
+            return false;
+        for (const conv of parsed) {
+            const messages = conv?.chat_messages;
+            if (!Array.isArray(messages))
+                continue;
+            for (const msg of messages) {
+                if (msg?.sender === "assistant") {
+                    return true;
+                }
+            }
+        }
+        return false;
+    }
+    if (source === "gemini_web") {
+        if (!Array.isArray(parsed))
+            return false;
+        for (const conv of parsed) {
+            const assistantTurns = conv?.assistant_turns;
+            if (typeof assistantTurns === "number" && assistantTurns > 0) {
+                return true;
+            }
+        }
+        return false;
+    }
+    if (source === "grok_web") {
+        const conversations = parsed?.conversations;
+        if (!Array.isArray(conversations))
+            return false;
+        for (const wrapper of conversations) {
+            const responses = wrapper?.responses;
+            if (!Array.isArray(responses))
+                continue;
+            for (const resp of responses) {
+                const sender = resp
+                    ?.response?.sender;
+                if (typeof sender === "string" && sender.toLowerCase() !== "human") {
+                    return true;
+                }
+            }
+        }
+        return false;
+    }
+    return true;
+}
+function summarizeSizes(sizes) {
+    if (sizes.length === 0) {
+        return { min: 0, max: 0, median: 0, total: 0 };
+    }
+    const sorted = [...sizes].sort((a, b) => a - b);
+    const mid = Math.floor(sorted.length / 2);
+    const median = sorted.length % 2 === 0
+        ? Math.floor((sorted[mid - 1] + sorted[mid]) / 2)
+        : sorted[mid];
+    return {
+        min: sorted[0],
+        max: sorted[sorted.length - 1],
+        median,
+        total: sizes.reduce((a, b) => a + b, 0),
+    };
+}
+//# sourceMappingURL=staging.js.map

package/dist/src/ingestion/staging.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"staging.js","sourceRoot":"","sources":["../../../src/ingestion/staging.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,OAAO,EAAE,MAAM,SAAS,CAAC;AACzB,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,MAAM,MAAM,aAAa,CAAC;AAEjC,OAAO,EAAE,cAAc,EAAqB,MAAM,oBAAoB,CAAC;AAEvE,6DAA6D;AAC7D,MAAM,CAAC,MAAM,gBAAgB,GAAG,GAAG,CAAC;AAEpC,mEAAmE;AACnE,MAAM,CAAC,MAAM,iBAAiB,GAA4C;IACxE,WAAW,EAAE,CAAC,OAAO,EAAE,MAAM,CAAC;IAC9B,UAAU,EAAE,CAAC,OAAO,EAAE,MAAM,CAAC;IAC7B,QAAQ,EAAE,CAAC,OAAO,EAAE,MAAM,CAAC;IAC3B,UAAU,EAAE,CAAC,OAAO,CAAC;CACtB,CAAC;AAsEF,MAAM,aAAa,GAAG,eAAe,CAAC;AACtC,MAAM,cAAc,GAAG,gBAAgB,CAAC;AACxC,MAAM,WAAW,GAAG,aAAa,CAAC;AAClC,MAAM,oBAAoB,GAAG,sBAAsB,CAAC;AAEpD,iFAAiF;AACjF,MAAM,sBAAsB,GAAG,IAAI,GAAG,CAAC;IACrC,aAAa;IACb,cAAc;IACd,WAAW;IACX,oBAAoB;CACrB,CAAC,CAAC;AAEH;;;;GAIG;AACH,MAAM,CAAC,MAAM,kBAAkB,GAAG,IAAI,GAAG,IAAI,CAAC;AAE9C;;;;;;GAMG;AACH,MAAM,CAAC,MAAM,yBAAyB,GAAG,GAAG,GAAG,IAAI,GAAG,IAAI,CAAC;AAE3D,mCAAmC;AACnC,MAAM,SAAS,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC;AAExD,qBAAqB;AAErB;;;GAGG;AACH,MAAM,UAAU,cAAc,CAAC,QAAgB;IAC7C,MAAM,OAAO,GAAG,EAAE,CAAC,WAAW,CAAC,QAAQ,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC;IAClE,OAAO,OAAO;SACX,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,EAAE,IAAI,CAAC,sBAAsB,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;SAChE,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC;SACvC,IAAI,EAAE,CAAC;AACZ,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,aAAa,CAAC,MAI7B;IACC,MAAM,EAAE,QAAQ,EAAE,GAAG,MAAM,CAAC;IAC5B,IAAI,CAAC,cAAc,CAAC,QAAQ,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC;QAC5C,MAAM,IAAI,KAAK,CAAC,mBAAmB,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC;IACtD,CAAC;IACD,MAAM,KAAK,GAAG,cAAc,CAAC,QAAQ,CAAC,CAAC,GAAG,CAAe,CAAC,CAAC,EAAE,EAAE;QAC7D,MAAM,IAAI,GAAG,EAAE,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;QAC5B,OAAO;YACL,IAAI,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC;YACtB,IAAI,EAAE,CAAC;YACP,UAAU,EAAE,IAAI,CAAC,IAAI;SACtB,CAAC;IACJ,CAAC,CAAC,CAAC;IACH,OAAO;QACL,UAAU,EAAE,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC;QACnC,UAAU,EAAE,CAAC,MAAM,CAAC,GAAG,IAAI,IAAI,IAAI,EAAE,CAAC,CAAC,WAAW,EAAE;QACpD,MAAM,EAAE,MAAM,CAAC,MAAM;QACrB,IAAI,EAAE,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC;QAC5B,KAAK;KACN,CAAC;AACJ,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,YAAY,CAAC,MAG5B;IACC,MAAM,EAAE,QAAQ,EAAE,GAAG,MAAM,CAAC;IAC5B,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC,iBAAiB,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC;IAE5D,oEAAoE;IACpE,MAAM,OAAO,GAAG,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,YAAY,CAAC,CAAC,EAAE,QAAQ,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC;IAErF,0DAA0D;IAC1D,MAAM,UAAU,GAAG,IAAI,GAAG,EAAkB,CAAC;IAC7C,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;QACxB,IAAI,CAAC,CAAC,YAAY,EAAE,CAAC;YACnB,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,YAAY,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QAC5E,CAAC;IACH,CAAC;IACD,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;QACxB,IAAI,CAAC,CAAC,YAAY,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC;YAChE,CAAC,CAAC,cAAc,GAAG,IAAI,CAAC;YACxB,IAAI,CAAC,CAAC,CAAC,YAAY,CAAC,QAAQ,CAAC,gBAAgB,CAAC,EAAE,CAAC;gBAC/C,CAAC,CAAC,YAAY,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;YACxC,CAAC;QACH,CAAC;IACH,CAAC;IAED,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC;IAC/C,MAAM,cAAc,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,MAAM,CAAC;IACjE,oEAAoE;IACpE,yEAAyE;IACzE,uEAAuE;IACvE,iEAAiE;IACjE,4CAA4C;IAC5C,MAAM,SAAS,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;IAC/D,MAAM,mBAAmB,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,MAAM,CAAC;IAC3E,MAAM,cAAc,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC;IAE/E,OAAO;QACL,UAAU,EAAE,QAAQ,CAAC,UAAU;QAC/B,MAAM,EAAE,QAAQ,CAAC,MAAM;QACvB,YAAY,EAAE,CAAC,MAAM,CAAC,GAAG,IAAI,IAAI,IAAI,EAAE,CAAC,CAAC,WAAW,EAAE;QACtD,WAAW,EAAE,OAAO,CAAC,MAAM;QAC3B,eAAe,EAAE,cAAc;QAC/B,iBAAiB,EAAE,OAAO,CAAC,MAAM,GAAG,cAAc;QAClD,iBAAiB,EAAE,cAAc,CAAC,KAAK,CAAC;QACxC,oBAAoB,EAAE,mBAAmB;QACzC,YAAY,EAAE;YACZ,SAAS;YACT,WAAW,EAAE,OAAO,CAAC,MAAM,GAAG,SAAS;SACxC;QACD,eAAe,EAAE,cAAc;QAC/B,KAAK,EAAE,OAAO;KACf,CAAC;AACJ,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,qBAAqB,CAAC,QAAgB,EAAE,SAA2B;IACjF,EAAE,CAAC,aAAa,CACd,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,aAAa,CAAC,EAClC,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,GAAG,IAAI,EAClD,OAAO,CACR,CAAC;IACF,EAAE,CAAC,aAAa,CACd,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,cAAc,CAAC,EACnC,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC,GAAG,IAAI,EACnD,OAAO,CACR,CAAC;AACJ,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,UAAU,CAAC,MAI1B;IACC,MAAM,QAAQ,GAAG,aAAa,CAAC,MAAM,CAAC,CAAC;IACvC,MAAM,SAAS,GAAG,MAAM,CAAC,GAAG;QAC1B,CAAC,CAAC,YAAY,CAAC,EAAE,QAAQ,EAAE,GAAG,EAAE,MAAM,CAAC,GAAG,EAAE,CAAC;QAC7C,CAAC,CAAC,YAAY,CAAC,EAAE,QAAQ,EAAE,CAAC,CAAC;IAC/B,qBAAqB,CAAC,MAAM,CAAC,QAAQ,EAAE,EAAE,QAAQ,EAAE,SAAS,EAAE,CAAC,CAAC;IAChE,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,CAAC;AACjC,CAAC;AAED,2BAA2B;AAE3B,SAAS,YAAY,CACnB,IAAkB,EAClB,MAAoB,EACpB,WAAwB;IAExB,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,WAAW,EAAE,CAAC;IAClD,MAAM,SAAS,GAAG,WAAW,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IACvC,MAAM,OAAO,GAAiB,EAAE,CAAC;IACjC,MAAM,KAAK,GAAa,EAAE,CAAC;IAE3B,IAAI,CAAC,SAAS;QAAE,OAAO,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC;IACnD,IAAI,IAAI,CAAC,UAAU,KAAK,CAAC;QAAE,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;SAC5C,IAAI,IAAI,CAAC,UAAU,GAAG,gBAAgB;QAAE,OAAO,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;IAEvE,sEAAsE;IACtE,oEAAoE;IACpE,sEAAsE;IACtE,MAAM,iBAAiB,GACrB,IAAI,CAAC,UAAU,GAAG,CAAC,IAAI,IAAI,CAAC,UAAU,IAAI,yBAAyB,CAAC;IACtE,MAAM,eAAe,GAAG,iBAAiB;QACvC,CAAC,CAAC,IAAI,CAAC,UAAU;QACjB,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,kBAAkB,EAAE,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC,CAAC;IAE/D,IAAI,MAAM,GAAkB,IAAI,CAAC;IACjC,IAAI,CAAC;QACH,MAAM,EAAE,GAAG,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;QACvC,IAAI,CAAC;YACH,MAAM,GAAG,GAAG,MAAM,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC;YAC1C,MAAM,IAAI,GAAG,GAAG,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,QAAQ,CAAC,EAAE,EAAE,GAAG,EAAE,CAAC,EAAE,GAAG,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YACzE,MAAM,GAAG,GAAG,CAAC,QAAQ,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;QACjC,CAAC;gBAAS,CAAC;YACT,EAAE,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;QACnB,CAAC;IACH,CAAC;IAAC,MAAM,CAAC;QACP,KAAK,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;IAC3B,CAAC;IAED,qEAAqE;IACrE,oEAAoE;IACpE,IAAI,WAAW,GAAkB,IAAI,CAAC;IACtC,IAAI,MAAM,EAAE,CAAC;QACX,IAAI,CAAC;YACH,WAAW,GAAG,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACpC,CAAC;QAAC,MAAM,CAAC;YACP,KAAK,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;QAC5B,CAAC;IACH,CAAC;IAED,wEAAwE;IACxE,IAAI,MAAM,GAAG,IAAI,CAAC;IAClB,MAAM,KAAK,GAAG,GAAG,KAAK,MAAM,IAAI,CAAC,MAAM,KAAK,IAAI,IAAI,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC;IAC1E,IAAI,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;QACrB,MAAM,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC;QAC7B,IAAI,CAAC,MAAM;YAAE,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;IACxC,CAAC;IAED,gBAAgB;IAChB,sEAAsE;IACtE,uEAAuE;IACvE,wEAAwE;IACxE,kEAAkE;IAClE,mEAAmE;IACnE,uCAAuC;IACvC,kEAAkE;IAClE,MAAM,SAAS,GAAG,MAAM,KAAK,IAAI,IAAI,MAAM,CAAC,MAAM,KAAK,IAAI,CAAC,UAAU,CAAC;IACvE,IAAI,MAAM,IAAI,SAAS,IAAI,IAAI,CAAC,UAAU,GAAG,CAAC,EAAE,CAAC;QAC/C,IAAI,GAAG,KAAK,OAAO,EAAE,CAAC;YACpB,IAAI,SAAS,EAAE,CAAC;gBACd,IAAI,MAAe,CAAC;gBACpB,IAAI,OAAO,GAAG,KAAK,CAAC;gBACpB,IAAI,CAAC;oBACH,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC;oBAC9C,OAAO,GAAG,IAAI,CAAC;gBACjB,CAAC;gBAAC,MAAM,CAAC;oBACP,KAAK,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;gBAC5B,CAAC;gBACD,IACE,OAAO;oBACP,sBAAsB,CAAC,MAAM,CAAC;oBAC9B,CAAC,gBAAgB,CAAC,MAAM,EAAE,MAAM,CAAC,EACjC,CAAC;oBACD,OAAO,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC;gBACrC,CAAC;YACH,CAAC;iBAAM,CAAC;gBACN,KAAK,CAAC,IAAI,CAAC,uBAAuB,CAAC,CAAC;YACtC,CAAC;QACH,CAAC;aAAM,IAAI,GAAG,KAAK,MAAM,EAAE,CAAC;YAC1B,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,EAAE,CAAC;gBAC1B,KAAK,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;YAC5B,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO;QACL,IAAI,EAAE,IAAI,CAAC,IAAI;QACf,IAAI,EAAE,IAAI,CAAC,IAAI;QACf,UAAU,EAAE,IAAI,CAAC,UAAU;QAC3B,SAAS,EAAE,GAAG;QACd,SAAS;QACT,OAAO,EAAE,MAAM;QACf,YAAY,EAAE,WAAW;QACzB,cAAc,EAAE,KAAK;QACrB,YAAY,EAAE,OAAO;QACrB,KAAK;KACN,CAAC;AACJ,CAAC;AAED,SAAS,WAAW,CAAC,CAAgB;IACnC,IAAI,CAAC,CAAC,CAAC,SAAS;QAAE,OAAO,KAAK,CAAC;IAC/B,IAAI,CAAC,CAAC,UAAU,GAAG,gBAAgB;QAAE,OAAO,KAAK,CAAC;IAClD,IAAI,CAAC,CAAC,CAAC,OAAO;QAAE,OAAO,KAAK,CAAC;IAC7B,IAAI,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC,aAAa,CAAC;QAAE,OAAO,KAAK,CAAC;IAClD,IAAI,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC,YAAY,CAAC;QAAE,OAAO,KAAK,CAAC;IACjD,mEAAmE;IACnE,oEAAoE;IACpE,IAAI,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC,uBAAuB,CAAC;QAAE,OAAO,KAAK,CAAC;IAC5D,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,QAAQ,CAAC,CAAS;IACzB,MAAM,IAAI,GAAG,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;IACzC,MAAM,EAAE,GAAG,EAAE,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;IAC/B,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,MAAM,CAAC,KAAK,CAAC,EAAE,GAAG,IAAI,CAAC,CAAC;QACpC,IAAI,IAAI,GAAG,CAAC,CAAC;QACb,OAAO,CAAC,IAAI,GAAG,EAAE,CAAC,QAAQ,CAAC,EAAE,EAAE,GAAG,EAAE,CAAC,EAAE,GAAG,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC;YAC9D,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,CAAC;QACrC,CAAC;IACH,CAAC;YAAS,CAAC;QACT,EAAE,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;IACnB,CAAC;IACD,OAAO,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;AAC5B,CAAC;AAED,SAAS,WAAW,CAAC,GAAW;IAC9B,IAAI,CAAC;QACH,IAAI,WAAW,CAAC,OAAO,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QACtD,OAAO,IAAI,CAAC;IACd,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC;AAED,SAAS,YAAY,CAAC,GAAW;IAC/B,OAAO,GAAG,CAAC,MAAM,IAAI,SAAS,CAAC,MAAM,IAAI,GAAG,CAAC,QAAQ,CAAC,CAAC,EAAE,SAAS,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;AAC/F,CAAC;AAED;;;;GAIG;AACH,SAAS,sBAAsB,CAAC,MAAoB;IAClD,OAAO,CACL,MAAM,KAAK,aAAa;QACxB,MAAM,KAAK,YAAY;QACvB,MAAM,KAAK,YAAY;QACvB,MAAM,KAAK,UAAU,CACtB,CAAC;AACJ,CAAC;AAED;;;;;GAKG;AACH,SAAS,gBAAgB,CAAC,MAAe,EAAE,MAAoB;IAC7D,IAAI,MAAM,KAAK,aAAa,EAAE,CAAC;QAC7B,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC;YAAE,OAAO,KAAK,CAAC;QACzC,KAAK,MAAM,IAAI,IAAI,MAAM,EAAE,CAAC;YAC1B,MAAM,OAAO,GAAI,IAAqC,EAAE,OAAO,CAAC;YAChE,IAAI,CAAC,OAAO,IAAI,OAAO,OAAO,KAAK,QAAQ;gBAAE,SAAS;YACtD,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,MAAM,CAAC,OAAkC,CAAC,EAAE,CAAC;gBACrE,MAAM,IAAI,GAAI,IAA6D;oBACzE,EAAE,OAAO,EAAE,MAAM,EAAE,IAAI,CAAC;gBAC1B,IAAI,IAAI,KAAK,WAAW;oBAAE,OAAO,IAAI,CAAC;YACxC,CAAC;QACH,CAAC;QACD,OAAO,KAAK,CAAC;IACf,CAAC;IACD,IAAI,MAAM,KAAK,YAAY,EAAE,CAAC;QAC5B,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC;YAAE,OAAO,KAAK,CAAC;QACzC,KAAK,MAAM,IAAI,IAAI,MAAM,EAAE,CAAC;YAC1B,MAAM,QAAQ,GAAI,IAA2C,EAAE,aAAa,CAAC;YAC7E,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC;gBAAE,SAAS;YACvC,KAAK,MAAM,GAAG,IAAI,QAAQ,EAAE,CAAC;gBAC3B,IAAK,GAAmC,EAAE,MAAM,KAAK,WAAW,EAAE,CAAC;oBACjE,OAAO,IAAI,CAAC;gBACd,CAAC;YACH,CAAC;QACH,CAAC;QACD,OAAO,KAAK,CAAC;IACf,CAAC;IACD,IAAI,MAAM,KAAK,YAAY,EAAE,CAAC;QAC5B,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC;YAAE,OAAO,KAAK,CAAC;QACzC,KAAK,MAAM,IAAI,IAAI,MAAM,EAAE,CAAC;YAC1B,MAAM,cAAc,GAAI,IAA6C,EAAE,eAAe,CAAC;YACvF,IAAI,OAAO,cAAc,KAAK,QAAQ,IAAI,cAAc,GAAG,CAAC,EAAE,CAAC;gBAC7D,OAAO,IAAI,CAAC;YACd,CAAC;QACH,CAAC;QACD,OAAO,KAAK,CAAC;IACf,CAAC;IACD,IAAI,MAAM,KAAK,UAAU,EAAE,CAAC;QAC1B,MAAM,aAAa,GAAI,MAA6C,EAAE,aAAa,CAAC;QACpF,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,aAAa,CAAC;YAAE,OAAO,KAAK,CAAC;QAChD,KAAK,MAAM,OAAO,IAAI,aAAa,EAAE,CAAC;YACpC,MAAM,SAAS,GAAI,OAA0C,EAAE,SAAS,CAAC;YACzE,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,SAAS,CAAC;gBAAE,SAAS;YACxC,KAAK,MAAM,IAAI,IAAI,SAAS,EAAE,CAAC;gBAC7B,MAAM,MAAM,GAAI,IAAmD;oBACjE,EAAE,QAAQ,EAAE,MAAM,CAAC;gBACrB,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,MAAM,CAAC,WAAW,EAAE,KAAK,OAAO,EAAE,CAAC;oBACnE,OAAO,IAAI,CAAC;gBACd,CAAC;YACH,CAAC;QACH,CAAC;QACD,OAAO,KAAK,CAAC;IACf,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,cAAc,CAAC,KAAe;IACrC,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACvB,OAAO,EAAE,GAAG,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC;IACjD,CAAC;IACD,MAAM,MAAM,GAAG,CAAC,GAAG,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAChD,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IAC1C,MAAM,MAAM,GACV,MAAM,CAAC,MAAM,GAAG,CAAC,KAAK,CAAC;QACrB,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,GAAG,GAAG,CAAC,CAAE,GAAG,MAAM,CAAC,GAAG,CAAE,CAAC,GAAG,CAAC,CAAC;QACnD,CAAC,CAAC,MAAM,CAAC,GAAG,CAAE,CAAC;IACnB,OAAO;QACL,GAAG,EAAE,MAAM,CAAC,CAAC,CAAE;QACf,GAAG,EAAE,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAE;QAC/B,MAAM;QACN,KAAK,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;KACxC,CAAC;AACJ,CAAC"}