@synap-core/cli 1.11.0 → 1.13.0

package/dist/commands/import.d.ts

@@ -1,22 +1,43 @@
 /**
- * synap import — "capture, but for files and URLs".
+ * synap import — "capture, but for files and URLs" (personal OS intake).
  *
- * Routes EVERY input (local file, folder, or http(s) URL) through the SAME AI
- * capture centerpiece as free-text `capture`:
- *   POST /capture/structure  (AI entity extraction + workspace/project routing)
- *   POST /capture/execute    (materialize the chosen proposals + relations)
+ * Routing (INTAKE-CONSOLIDATION-PLAN Wave 1):
+ *   - Superwhisper store-first     → lib/superwhisper-import (unchanged)
+ *   - Single file OR single URL    → POST /capture/structure + /capture/execute
+ *   - Any .csv (even ONE)          → its own `source: csv` batch on the table
+ *                                    door below (a CSV is a table, never prose)
+ *   - 2 text-like files
+ *     (md / markdown / txt / csv)  → ONE batch via SYNCHRONOUS REST import:
+ *         POST /import/analyze  → human review (or --yes / --dry-run)
+ *         POST /import/apply    → materialize the EXACT previewed ops
+ *   - ≥CORPUS_THRESHOLD (3) such
+ *     files                        → ONE BACKGROUND job (the sync door times out):
+ *         POST /import/enqueue-corpus  → { jobId }, returns immediately
+ *         GET  /import/corpus-job/{id} → poll to "completed"
+ *       On completion the result is ONE pending governed `import.graph`
+ *       proposal — NOT stored entities. `--dry-run` never takes this door
+ *       (enqueueing is itself a write).
+ *   - Mixed binary/URL + text      → capture path for URLs/binaries;
+ *                                    batch text via /import/* when N≥2
  *
- * One item per file/url. This inherits AI entity extraction, the relation graph,
- * workspace routing AND project routing for free — no separate /import/* path.
+ * Directory walks cap at DIR_FILE_CAP (sync analyze max); truncate is warned,
+ * never silent. After expansion, batch-eligible text is also capped at
+ * GLOBAL_BATCH_CAP (same ceiling) across all inputs — not only per-directory.
+ * Workspace/project from --workspace / --project (or cfg);
+ * session from --session (active session id).
  *
- * Inputs:
- *   - http(s) URL              → sent as `url`
- *   - directory                → recursively globbed (skips node_modules/.git/dotfiles)
- *   - file                     → text-like → utf8, everything else → base64
+ * Optional --home-map rewrites batch item paths so the backend path heuristic
+ * (workspace **name** as a path segment) pins multi-home destinations. Example:
+ *   --home-map "Projects=Builder,Posts=Content OS"
+ * rewrites `5. Projects/foo.md` → `Builder/5. Projects/foo.md`. No product
+ * defaults — user-supplied pairs only. It applies to the corpus door too: the
+ * background job forwards `items` verbatim into the SAME
+ * `ImportOrchestrator.analyzeLarge` the sync door reaches, so the path
+ * heuristic is identical on both.
  *
- * Routing: --workspace / --project override the AI's suggestion; otherwise the
- * AI-proposed targetWorkspaceId / targetProjectId from /capture/structure win.
+ * No Company OS coupling — pure personal intake doors.
  */
+import { type StructureResult, type ExecuteResult } from "../lib/capture-structure.js";
 export interface ImportOpts {
     workspace?: string;
     project?: string;
@@ -26,13 +47,372 @@ export interface ImportOpts {
     json?: boolean;
     podUrl?: string;
     apiKey?: string;
+    /**
+     * Preferred-home map: comma-separated `pathSubstring=workspaceNameOrId`.
+     * Matched paths (case-insensitive substring) are rewritten so the first
+     * path segment is the resolved workspace **name** — backend deep structure
+     * matches workspace names in path segments.
+     */
+    homeMap?: string;
     /** Adapter name: "superwhisper" enables paired store-first path. */
     source?: string;
     /** Skip AI structure; store corpus units (Superwhisper). */
     storeFirst?: boolean;
     withAudio?: boolean;
+    /**
+     * Preserve the ORIGINAL file alongside the entities the AI derived from it.
+     *
+     * Default (absent) is extract-and-discard: the bytes are read for structuring
+     * and then dropped, so an imported PDF produced entities with no `documentId`
+     * and no source document — dogfooding an import of a real PDF found exactly
+     * that. `--with-audio` is NOT this: it is Superwhisper-adapter-specific and
+     * only rides the store-first path. This flag threads `keepRaw` + the bytes
+     * through the SAME `capture.execute` disposition the browser uses.
+     */
+    keepRaw?: boolean;
     limit?: number;
     concurrency?: number;
     resume?: boolean;
+    /** `--job-status <uuid>`: re-poll a background corpus job instead of importing. */
+    jobStatus?: string;
 }
+/**
+ * At or above this many batch-eligible text files, route to the BACKGROUND
+ * corpus door (`POST /import/enqueue-corpus`) instead of the synchronous
+ * `/import/analyze`.
+ *
+ * Why 3, measured rather than guessed: 3 markdown files complete in ~2m01s on
+ * the background worker, and the SAME 3 files fail 0/5 through /import/analyze,
+ * each dying at 45–48s against the pod's 45s request timeout. So 3 is the
+ * smallest N at which the synchronous door is observed to be unusable.
+ *
+ * Below it (1–2 files) the synchronous path stays: it is proven to work at that
+ * size and it is strictly more interactive — it renders the operation list, the
+ * multi-home placement and the quality report, then applies on a y/N confirm.
+ * The corpus door can offer none of that; it returns a job handle and files ONE
+ * governed proposal for later review. Routing everything through it would trade
+ * a working interactive preview for a background job on inputs that never
+ * needed one.
+ */
+export declare const CORPUS_THRESHOLD = 3;
+type ImportSource = "obsidian" | "markdown" | "csv";
+type ImportItem = {
+    kind: "url";
+    label: string;
+    url: string;
+} | {
+    kind: "file";
+    label: string;
+    /** Source-relative path for /import/analyze (vault structure). */
+    path: string;
+    file: {
+        content: string;
+        mimeType: string;
+        filename: string;
+        encoding: "base64" | "utf8";
+    };
+};
+/** Wire shape from POST /import/analyze (Wave 1 REST → ImportOrchestrator). */
+interface ImportAnalyzeResult {
+    workspaceId?: string | null;
+    source?: string;
+    mode?: string;
+    proposalId?: string | null;
+    sessionId?: string | null;
+    operations?: unknown[];
+    summary?: string;
+    stats?: Record<string, unknown>;
+    droppedReferences?: number;
+    aiTyped?: number;
+    tablePlan?: unknown;
+    /** Multi-home placement summary (create_entity destinations). */
+    homes?: {
+        byWorkspace?: Record<string, number>;
+        podWide?: number;
+        byProject?: Record<string, number>;
+        multiHome?: boolean;
+    };
+    /** Continuous-improvement quality report (stored on proposal too). */
+    quality?: {
+        score?: number;
+        summary?: string;
+        counts?: Record<string, unknown>;
+        hierarchy?: Record<string, unknown>;
+        findings?: Array<{
+            id?: string;
+            severity?: string;
+            message?: string;
+        }>;
+        nextUpgrades?: string[];
+    };
+    [key: string]: unknown;
+}
+/** Wire shape from POST /import/apply. */
+interface ImportApplyResult {
+    workspaceId?: string | null;
+    source?: string;
+    created?: number;
+    linked?: number;
+    [key: string]: unknown;
+}
+/**
+ * Which batch-eligible files go through the table/batch door, and in which
+ * batches:
+ *   - every CSV, even a lone one, in ONE `source: csv` batch. A CSV is a TABLE:
+ *     the batch door builds a column→kind plan for it, while the single-file
+ *     capture door reads it as free text (a 1,500-row contacts export came back
+ *     as one degraded note filed in the wrong space). Keeping CSVs apart also
+ *     stops a mixed md + csv batch from parsing the markdown as CSV.
+ *   - the other text files in ONE batch when there are ≥2; a lone one stays on
+ *     the capture door (unchanged).
+ */
+export declare function groupBatchFiles<T extends {
+    path: string;
+}>(files: T[]): {
+    batches: T[][];
+    single: T[];
+};
+type HomeMapPair = {
+    pathSubstring: string;
+    workspace: string;
+};
+/** Resolved map: path substring → workspace **name** (for path prefix rewrite). */
+type ResolvedHomeMapEntry = {
+    pathSubstring: string;
+    workspaceName: string;
+};
+/**
+ * Parse `--home-map "Projects=Builder,Posts=Content OS"` into pairs.
+ * Values may be workspace names or UUIDs (resolved later).
+ */
+export declare function parseHomeMap(raw: string): HomeMapPair[];
+/**
+ * If `path` contains any map pathSubstring (case-insensitive), prefix with the
+ * resolved workspace name so deep-structure segment match pins that home.
+ * First matching pair wins. Skips rewrite when the path already starts with the
+ * workspace name segment.
+ */
+export declare function applyHomeMapToPath(path: string, map: ResolvedHomeMapEntry[]): string;
+export interface ItemOutcome {
+    label: string;
+    status: "stored" | "proposed" | "degraded" | "dry-run" | "skipped" | "empty" | "failed"
+    /**
+     * Background corpus job accepted, but not observed to finish inside this
+     * invocation's wait window. Nothing was lost — the job keeps running on the
+     * pod. Exit code stays 0 (see importExitCode): a still-running job is not a
+     * degraded one.
+     */
+     | "queued";
+    entities?: number;
+    relations?: number;
+    workspaceId?: string | null;
+    projectId?: string | null;
+    error?: string;
+    /**
+     * Why the item degraded. Either a pod plumbing reason (`is_auth_error` |
+     * `is_invalid_response` | `is_empty_result`) or an Intelligence Service
+     * extraction reason (`vision_provider_not_configured`,
+     * `pdf_scanned_needs_ocr`, `unsupported_type`, …) — the pod no longer
+     * relabels the latter as the former. The human path prints this through
+     * `degradedMessage()`; carrying the raw token here keeps `--json` machine
+     * readable while the human line stays token-free.
+     */
+    degradedReason?: string;
+    /**
+     * What happened to the ORIGINAL file under `--keep-raw`. Absent when nothing
+     * was kept. Never collapsed to a boolean: `denied` (policy said no) and
+     * `failed` (storage broke) are different problems, and `proposed` means the
+     * blob is still only a pending review.
+     */
+    sourceFile?: "stored" | "proposed" | "denied" | "failed";
+    /**
+     * Where a person reviews a `proposed` import (the pod's /open/<id> page).
+     * Present when the import was filed and NOT applied — an agent credential
+     * cannot approve, so its import stops at the proposal.
+     */
+    reviewUrl?: string;
+    /** Present for the multi-file /import/* batch path. */
+    batch?: {
+        source: ImportSource;
+        fileCount: number;
+        proposalId?: string | null;
+        sessionId?: string | null;
+        analyze?: ImportAnalyzeResult;
+        apply?: ImportApplyResult;
+        /** Present for the background /import/enqueue-corpus path. */
+        corpus?: {
+            jobId: string | null;
+            /** Last pg-boss state we observed, or "unknown" if we never got one. */
+            state: string;
+            /** True when the wait window elapsed with the job still running. */
+            timedOut: boolean;
+            waitedMs: number;
+            itemCount: number;
+            /**
+             * File-level truth from the job's own output. `outcomeKnown: false` means
+             * the pod did not report it (older pod, or job unfinished) — a script must
+             * treat that as UNKNOWN, not as "0 failed".
+             */
+            outcomeKnown?: boolean;
+            filesProcessed?: number | null;
+            filesFailed?: number | null;
+            findings?: Array<{
+                id?: string;
+                severity?: string;
+                message?: string;
+            }>;
+            proposalId?: string | null;
+        };
+    };
+}
+/**
+ * Build the outcome for an item the IS structurer could not structure.
+ *
+ * Carries `degradedReason` so `--json` reports the same cause the human path
+ * already prints via `degradedMessage()`. Without it a scripted import saw
+ * `{"status":"degraded"}` with no way to tell an auth error from an empty
+ * result — the reason existed on the wire and was dropped at this one seam.
+ */
+export declare function degradedOutcome(label: string, res: StructureResult, workspaceId: string | null, projectId: string | null): ItemOutcome;
+/**
+ * The exit code for a finished import run.
+ *
+ * A degraded item produced ZERO entities — the IS structurer was down and the
+ * CLI deliberately created nothing. Exiting 0 on that told a scripted backfill
+ * "success" while most of its input was silently lost, so `degraded` is a
+ * non-zero outcome exactly like `failed`.
+ *
+ * Everything else stays 0, including a clean `--dry-run` (nothing was meant to
+ * be written), a `skipped` item (the user declined) and an `empty` one (the
+ * structurer ran fine and honestly found nothing to create).
+ *
+ * Scoped to `import` on purpose: this is `import`'s own final line, not a
+ * shared helper, so no other command's exit semantics change.
+ */
+export declare function importExitCode(outcomes: Array<{
+    status: ItemOutcome["status"];
+}>): number;
+/**
+ * The `keepRaw` half of a `/capture/execute` body — `{}` unless `--keep-raw`
+ * was passed for a FILE item.
+ *
+ * Three things it must get right, all of which the pod silently tolerates:
+ *  • `content` MUST be base64. `fileToItem` reads text-like files as utf8, and
+ *    the pod does `Buffer.from(content, "base64")` unconditionally — sending
+ *    utf8 there stores a mangled blob with no error.
+ *  • `extractedText` is echoed from the structure response so the stored
+ *    document lands with a real v1 body. Skipping it leaves
+ *    `document_versions.content` empty, which blinds the embedding worker, the
+ *    retrieval body join and Typesense enrichment.
+ *  • A URL item has no bytes to keep, so `keepRaw` is simply not sent — the pod
+ *    would accept `keepRaw: true` with no file and do nothing.
+ */
+export declare function buildKeepRawPayload(opts: Pick<ImportOpts, "keepRaw">, item: ImportItem, structureRes: Pick<StructureResult, "extraction">): Record<string, unknown>;
+/**
+ * One line describing what ACTUALLY happened to a kept original. The pod
+ * reports four dispositions and they are not interchangeable: a
+ * governance-parked attach is not a save, and a policy denial is not a storage
+ * failure. Returns null when nothing was kept (so nothing is claimed).
+ */
+export declare function keepRawLine(res: ExecuteResult): string | null;
+/**
+ * The finished run's FILE-LEVEL outcome, as the pod stores it on the pg-boss job
+ * (`output`) and returns it from GET /import/corpus-job/{jobId}.
+ *
+ * This is the only place a partially failed corpus import is visible. The pod
+ * records `filesFailed` on the resulting import.graph proposal; before the pod
+ * returned this object the CLI could not see it and printed `0 failed` for a run
+ * that had dropped 2 of 3 files.
+ */
+export interface CorpusJobOutput {
+    proposalId?: string | null;
+    workspaceId?: string | null;
+    filesProcessed?: number;
+    filesFailed?: number;
+    qualityScore?: number;
+    findings?: Array<{
+        id?: string;
+        severity?: string;
+        message?: string;
+    }>;
+}
+/** What a completed corpus job actually did, once the wire value is judged. */
+export type CorpusFileOutcome = {
+    known: false;
+} | {
+    known: true;
+    filesProcessed: number | null;
+    filesFailed: number;
+};
+/**
+ * Judge a completed job's `output`.
+ *
+ * The ONE place absence is turned into `known: false`. A pod that predates the
+ * result-carrying worker returns no output; so does a job whose handler reported
+ * nothing. Neither is evidence that every file landed, so neither may collapse
+ * into `filesFailed: 0`.
+ */
+export declare function readCorpusFileOutcome(output: CorpusJobOutput | null | undefined): CorpusFileOutcome;
+/**
+ * The outcome status a COMPLETED corpus job earns.
+ *
+ * The ONE decision point: files were dropped → `degraded`, which `importExitCode`
+ * already treats as non-zero (no second exit-code path). An unknown outcome stays
+ * `proposed` — the proposal genuinely exists — but the caller must print
+ * CORPUS_OUTCOME_UNAVAILABLE so the unknown is never read as "0 failed".
+ */
+export declare function corpusCompletionOutcome(output: CorpusJobOutput | null | undefined): {
+    status: "proposed" | "degraded";
+    degradedReason?: string;
+};
+/** The message shown when the pod cannot tell us what happened per file. */
+export declare const CORPUS_OUTCOME_UNAVAILABLE: string;
+/**
+ * Human lines for a completed corpus run's file-level outcome.
+ *
+ * Exported so the run summary, the `--job-status` re-poll and the tests all read
+ * the SAME judgement — a second copy is how "0 failed" survived on one path.
+ */
+export declare function corpusOutcomeLines(output: CorpusJobOutput | null | undefined, itemCount?: number): string[];
+/**
+ * Which batch door to use. The ONE place the threshold is applied — the human
+ * message, the --json payload and the actual POST all read this.
+ *
+ * `--dry-run` always stays on `/import/analyze`: the corpus door has no preview
+ * mode (`previewOnly` is an /import/analyze field), and enqueueing is itself a
+ * durable write. A dry run that queued a background job would be a dry run that
+ * writes — the exact thing the flag promises not to do.
+ */
+export declare function chooseBatchDoor(args: {
+    fileCount: number;
+    dryRun?: boolean;
+}): "analyze" | "enqueue-corpus";
+/**
+ * How long to wait for a corpus job before handing the terminal back.
+ *
+ * Scaled from measured throughput (~45s/file) so a small corpus is not waited on
+ * for an hour and a 257-file backfill is not abandoned after five minutes, then
+ * clamped: never under 10 minutes, never over 2 hours. The ceiling is safe
+ * precisely because expiry is not failure — see corpusTimeoutLines().
+ */
+export declare function corpusPollTimeoutMs(itemCount: number): number;
+/**
+ * What we print when the wait window elapses.
+ *
+ * The job is STILL RUNNING — pg-boss holds it, the worker keeps chunking, and
+ * the governed `import.graph` proposal will still be written. Saying "failed"
+ * (or exiting non-zero) here would send a user to re-run a 257-file import that
+ * is already half-done. So: state plainly that nothing was lost, and give the
+ * exact command to re-poll.
+ */
+export declare function corpusTimeoutLines(jobId: string, waitedMs: number): string[];
+/**
+ * Re-poll a corpus job started by an earlier `synap import`.
+ *
+ * This is the command `corpusTimeoutLines()` tells the user to run, so it has to
+ * exist: a timeout message naming a command that does not resolve is worse than
+ * no message. One GET, no waiting — it reports the current state and exits.
+ */
+export declare function importJobStatus(jobId: string, opts: ImportOpts): Promise<void>;
 export declare function importData(inputs: string[], opts: ImportOpts): Promise<void>;
+export {};