@soundbi/sound-connect 0.1.0

package/dist/ingest.d.ts

@@ -0,0 +1,253 @@
+/**
+ * File ingestion logic for the Sound Connect bridge (STORY-011, STORY-012, STORY-013).
+ *
+ * STORY-011 implements:
+ *   - Path validation and confinement (no directory traversal — AC3)
+ *   - Markdown file reading and normalization (AC1)
+ *   - Content chunking for large files (AC1)
+ *   - SHA-256 content hash for idempotency (AC2, ADR-004)
+ *   - Provenance attachment and POST to /ingest/:slug (AC2)
+ *   - Summary return: chunks ingested, deduped count (AC4)
+ *
+ * STORY-012 extends to:
+ *   - Folder enumeration supporting .md and transcript text files (.txt/.vtt/.srt)
+ *   - Per-file source_type reflecting the file kind (markdown vs transcript)
+ *   - Idempotency across folder re-runs (sha256 already present → deduped)
+ *   - Per-file result table (ingested / deduped / failed)
+ *
+ * STORY-013 extends to:
+ *   - When `queueOnFailure` is enabled, network errors and 5xx responses write the
+ *     chunk to the local retry queue instead of throwing (ADR-011).
+ *   - 4xx responses still throw immediately (permanent auth/access error — queue won't help).
+ *   - IngestSummary gains a `chunks_queued` field reporting items sent to the retry queue.
+ *
+ * ADR-006: v1 handles text/markdown and transcript plaintext only. Binary formats
+ *          (PDF, .xlsx, .docx) deferred to v1.1.
+ * ADR-004: sha256 of normalized content is the idempotency key.
+ * ADR-011: All errors are thrown with descriptive messages; callers surface them
+ *          as MCP tool errors — never silent failures.
+ */
+/**
+ * File extensions accepted for ingestion (ADR-006: text/markdown + transcript plaintext).
+ * Binary formats (PDF, .xlsx, .docx) are deferred to v1.1.
+ */
+export declare const SUPPORTED_EXTENSIONS: Set<string>;
+/**
+ * Derive the source_type provenance value from a file extension (STORY-012, ADR-004).
+ * Returns 'markdown' for .md files, 'transcript' for .txt/.vtt/.srt.
+ */
+export declare function sourceTypeForExt(ext: string): 'markdown' | 'transcript';
+export interface IngestFileOptions {
+    /** Absolute or relative path to the markdown file (AC3: validated/confined). */
+    filePath: string;
+    /** Backend base URL (e.g. https://my-forge.azurecontainerapps.io). */
+    backendUrl: string;
+    /** Bound client slug — all ingestion is scoped to this client (ADR-005). */
+    clientSlug: string;
+    /** Bearer token for the authenticated peer (ADR-003). */
+    token: string;
+    /** Optional workstream slug to scope the ingested knowledge. */
+    workstreamSlug?: string;
+    /**
+     * Author email for provenance — extracted from the bearer token claim (AC2).
+     * If not supplied, the backend will derive it from the token.
+     * Passing it explicitly allows the backend to record accurate provenance even
+     * when the token's email claim uses a different casing convention.
+     */
+    authorEmail?: string;
+    /**
+     * STORY-013 / ADR-011: When true, network errors and 5xx responses write the
+     * failed chunk to the local retry queue instead of throwing. The summary will
+     * include a non-zero `chunks_queued` count. 4xx responses still throw (permanent).
+     *
+     * Defaults to false for backward compatibility with existing callers.
+     */
+    queueOnFailure?: boolean;
+}
+export interface IngestSummary {
+    /** Total number of chunks sent to the backend (includes queued). */
+    chunks_sent: number;
+    /** Number of chunks the backend accepted as new (not deduped). */
+    chunks_ingested: number;
+    /** Number of chunks the backend reported as duplicates (sha256 already present). */
+    chunks_deduped: number;
+    /**
+     * STORY-013: Number of chunks written to the local retry queue due to
+     * transient backend failure (network error or 5xx). Zero when queueOnFailure
+     * is not enabled or when all chunks succeeded.
+     */
+    chunks_queued: number;
+    /** Resolved absolute path of the file that was ingested. */
+    file: string;
+    /** SHA-256 of the full normalized content (hex). */
+    content_hash: string;
+}
+/** Status of a single file in a folder ingest run. */
+export type FileIngestStatus = 'ingested' | 'deduped' | 'failed';
+/** Per-file result in a folder ingest (STORY-012 AC4). */
+export interface FileIngestResult {
+    /** Absolute path of the file. */
+    file: string;
+    /** Outcome for this file. */
+    status: FileIngestStatus;
+    /** For ingested/deduped: number of chunks sent. */
+    chunks_sent?: number;
+    /** For ingested: number of chunks that were new. */
+    chunks_ingested?: number;
+    /** For deduped: number of chunks that were duplicates. */
+    chunks_deduped?: number;
+    /** For failed: the error message. */
+    error?: string;
+}
+/** Result of ingest_folder (STORY-012 AC4). */
+export interface FolderIngestResult {
+    /** Absolute resolved path of the folder. */
+    folder: string;
+    /** Total files found matching the glob/extension filter. */
+    files_found: number;
+    /** Number of files with at least one new chunk. */
+    files_ingested: number;
+    /** Number of files fully deduped (all chunks already present). */
+    files_deduped: number;
+    /** Number of files that failed during ingestion. */
+    files_failed: number;
+    /** Per-file results (STORY-012 AC4). */
+    results: FileIngestResult[];
+}
+/** Options for ingest_folder (STORY-012). */
+export interface IngestFolderOptions {
+    /** Path to the folder to enumerate. */
+    folderPath: string;
+    /**
+     * Optional glob pattern (extension filter). Defaults to all supported
+     * extensions: .md, .txt, .vtt, .srt (ADR-006).
+     * Only the extension portion of the pattern is matched — pass e.g. "*.md"
+     * or "*.vtt" to restrict to a specific type. Pass undefined for all types.
+     */
+    glob?: string;
+    /** Backend base URL. */
+    backendUrl: string;
+    /** Bound client slug (ADR-005). */
+    clientSlug: string;
+    /** Bearer token (ADR-003). */
+    token: string;
+    /** Optional workstream slug. */
+    workstreamSlug?: string;
+    /** Author email for provenance. */
+    authorEmail?: string;
+}
+/**
+ * Resolves and validates a single ingestable file path.
+ *
+ * Rules enforced:
+ *   1. Path must not be empty.
+ *   2. File must exist and be a regular file (not a directory or symlink loop).
+ *   3. File extension must be in SUPPORTED_EXTENSIONS (ADR-006: .md/.txt/.vtt/.srt).
+ *   4. File size must not exceed MAX_FILE_BYTES (ADR-004 body limit).
+ *
+ * Traversal confinement: `resolve()` normalises `../` sequences so the resolved
+ * path is always absolute. We do NOT restrict to a whitelist directory because
+ * peers legitimately have notes and transcripts anywhere on their machine. We do
+ * block the obvious attack vectors: empty paths, non-files, unsupported extensions.
+ *
+ * @param rawPath  The path supplied by the tool caller.
+ * @returns Resolved absolute path.
+ * @throws Error with a descriptive message on any validation failure.
+ */
+export declare function validateAndResolvePath(rawPath: string): Promise<string>;
+/**
+ * Parse the `glob` argument from ingest_folder into a Set of allowed extensions.
+ *
+ * Accepts patterns like "*.md", "*.vtt", "*.txt", "*.srt".
+ * If the pattern matches a known SUPPORTED_EXTENSIONS entry, only that extension
+ * is returned. If glob is undefined/empty, all SUPPORTED_EXTENSIONS are returned.
+ * If glob specifies an unsupported extension, throws with a clear message.
+ *
+ * This is intentionally simple — STORY-012 AC1 says "glob?" but the only
+ * meaningful variation is filtering by extension, not full glob matching.
+ * Full glob support (**, recursive patterns) is deferred to v1.1.
+ */
+export declare function parseGlobToExtensions(glob: string | undefined): Set<string>;
+/**
+ * Enumerate all ingestable files in a folder (non-recursive, flat listing).
+ *
+ * Returns absolute paths of files whose extension is in the allowed set.
+ * Silently skips subdirectories and files with unsupported extensions.
+ * Throws if the folder does not exist or is not a directory.
+ *
+ * STORY-012 AC1: enumerates .md and transcript text files (.txt/.vtt/.srt).
+ * Recursion is not in scope for v1 — flat directory only.
+ *
+ * @param folderPath   Absolute path to the folder to enumerate.
+ * @param allowedExts  Set of lowercase extensions to include (from parseGlobToExtensions).
+ * @returns Sorted list of absolute file paths.
+ */
+export declare function enumerateFolder(folderPath: string, allowedExts: Set<string>): Promise<string[]>;
+/**
+ * Normalize markdown content before hashing and chunking.
+ *
+ * Normalization steps:
+ *   1. Normalize line endings to LF.
+ *   2. Strip trailing whitespace from each line.
+ *   3. Collapse runs of 3+ blank lines to 2 blank lines (preserve paragraph spacing).
+ *   4. Trim leading/trailing blank lines from the document.
+ *
+ * This ensures that whitespace-only edits do not produce new sha256 hashes
+ * (no spurious re-ingestion) while preserving meaningful structure.
+ */
+export declare function normalizeMarkdown(content: string): string;
+/**
+ * Split normalized content into overlapping chunks for retrieval-friendly storage.
+ *
+ * Strategy: paragraph-aware sliding window.
+ *   - Split on double-newlines (paragraph boundaries) to avoid cutting mid-sentence.
+ *   - Accumulate paragraphs until the chunk would exceed CHUNK_SIZE_CHARS.
+ *   - Start the next chunk with the last CHUNK_OVERLAP_CHARS of the previous chunk
+ *     (approximate — overlap at paragraph boundary nearest to the overlap target).
+ *
+ * If the entire content fits in one chunk, returns a single-element array.
+ */
+export declare function chunkContent(content: string): string[];
+/** Compute the SHA-256 hex digest of a string. */
+export declare function sha256(content: string): string;
+/**
+ * Ingest a single local text file (markdown or transcript) into the Sound Connect corpus.
+ *
+ * Full pipeline:
+ *   1. Validate path — supports .md, .txt, .vtt, .srt (STORY-012 AC1, ADR-006).
+ *   2. Read file.
+ *   3. Normalize content (line endings, trailing whitespace, blank lines).
+ *   4. Chunk if large.
+ *   5. Hash each chunk (ADR-004 idempotency key).
+ *   6. Attach provenance with source_type derived from extension (STORY-012 AC2).
+ *   7. POST each chunk to /ingest/:slug.
+ *      STORY-013: When queueOnFailure=true, network errors and 5xx responses write
+ *      the chunk to the local retry queue instead of throwing. 4xx errors still throw.
+ *   8. Return summary (includes chunks_queued when queueOnFailure is used).
+ *
+ * @throws Error with a descriptive message on any failure (ADR-011).
+ */
+export declare function ingestMarkdownFile(opts: IngestFileOptions): Promise<IngestSummary>;
+/**
+ * Bulk-ingest a directory of markdown and transcript text files.
+ *
+ * Pipeline (STORY-012):
+ *   1. Resolve and validate the folder path.
+ *   2. Parse glob to determine which extensions to include (AC1).
+ *   3. Enumerate matching files in the folder (flat, non-recursive).
+ *   4. For each file: run through the ingestMarkdownFile pipeline (STORY-011 path).
+ *      - deduped = all chunks were already present (idempotency, AC3).
+ *      - ingested = at least one new chunk was accepted.
+ *      - failed   = any error during read/post.
+ *   5. Return per-file result table + folder-level totals (AC4).
+ *
+ * Idempotency (AC3): re-running the folder produces files_ingested=0 and
+ * files_deduped=N (one per file) because sha256 keys are already present.
+ *
+ * ADR-011: per-file failures are captured in the result table and do NOT abort
+ * the rest of the run — callers see which files failed and why.
+ *
+ * @throws Error only if the folder itself cannot be accessed (not per-file errors).
+ */
+export declare function ingestFolder(opts: IngestFolderOptions): Promise<FolderIngestResult>;
+//# sourceMappingURL=ingest.d.ts.map

package/dist/ingest.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"ingest.d.ts","sourceRoot":"","sources":["../src/ingest.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AA8BH;;;GAGG;AACH,eAAO,MAAM,oBAAoB,aAA2C,CAAC;AAE7E;;;GAGG;AACH,wBAAgB,gBAAgB,CAAC,GAAG,EAAE,MAAM,GAAG,UAAU,GAAG,YAAY,CAEvE;AAID,MAAM,WAAW,iBAAiB;IAChC,gFAAgF;IAChF,QAAQ,EAAE,MAAM,CAAC;IACjB,sEAAsE;IACtE,UAAU,EAAE,MAAM,CAAC;IACnB,4EAA4E;IAC5E,UAAU,EAAE,MAAM,CAAC;IACnB,yDAAyD;IACzD,KAAK,EAAE,MAAM,CAAC;IACd,gEAAgE;IAChE,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB;;;;;OAKG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB;;;;;;OAMG;IACH,cAAc,CAAC,EAAE,OAAO,CAAC;CAC1B;AAED,MAAM,WAAW,aAAa;IAC5B,oEAAoE;IACpE,WAAW,EAAE,MAAM,CAAC;IACpB,kEAAkE;IAClE,eAAe,EAAE,MAAM,CAAC;IACxB,oFAAoF;IACpF,cAAc,EAAE,MAAM,CAAC;IACvB;;;;OAIG;IACH,aAAa,EAAE,MAAM,CAAC;IACtB,4DAA4D;IAC5D,IAAI,EAAE,MAAM,CAAC;IACb,oDAAoD;IACpD,YAAY,EAAE,MAAM,CAAC;CACtB;AAID,sDAAsD;AACtD,MAAM,MAAM,gBAAgB,GAAG,UAAU,GAAG,SAAS,GAAG,QAAQ,CAAC;AAEjE,0DAA0D;AAC1D,MAAM,WAAW,gBAAgB;IAC/B,iCAAiC;IACjC,IAAI,EAAE,MAAM,CAAC;IACb,6BAA6B;IAC7B,MAAM,EAAE,gBAAgB,CAAC;IACzB,mDAAmD;IACnD,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,oDAAoD;IACpD,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,0DAA0D;IAC1D,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,qCAAqC;IACrC,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,+CAA+C;AAC/C,MAAM,WAAW,kBAAkB;IACjC,4CAA4C;IAC5C,MAAM,EAAE,MAAM,CAAC;IACf,4DAA4D;IAC5D,WAAW,EAAE,MAAM,CAAC;IACpB,mDAAmD;IACnD,cAAc,EAAE,MAAM,CAAC;IACvB,kEAAkE;IAClE,aAAa,EAAE,MAAM,CAAC;IACtB,oDAAoD;IACpD,YAAY,EAAE,MAAM,CAAC;IACrB,wCAAwC;IACxC,OAAO,EAAE,gBAAgB,EAAE,CAAC;CAC7B;AAED,6CAA6C;AAC7C,MAAM,WAAW,mBAAmB;IAClC,uCAAuC;IACvC,UAAU,EAAE,MAAM,CAAC;IACnB;;;;;OAKG;IACH,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,wBAAwB;IACxB,UAAU,EAAE,MAAM,CAAC;IACnB,mCAAmC;IACnC,UAAU,EAAE,MAAM,CAAC;IACnB,8BAA8B;IAC9B,KAAK,EAAE,MAAM,CAAC;IACd,gCAAgC;IAChC,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,mCAAmC;IACnC,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAID;;;;;;;;;;;;;;;;;GAiBG;AACH,wBAAsB,sBAAsB,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAkD7E;AAID;;;;;;;;;;;GAWG;AACH,wBAAgB,qBAAqB,CAAC,IAAI,EAAE,MAAM,GAAG,SAAS,GAAG,GAAG,CAAC,MAAM,CAAC,CAsB3E;AAED;;;;;;;;;;;;;GAaG;AACH,wBAAsB,eAAe,CACnC,UAAU,EAAE,MAAM,EAClB,WAAW,EAAE,GAAG,CAAC,MAAM,CAAC,GACvB,OAAO,CAAC,MAAM,EAAE,CAAC,CA4BnB;AAID;;;;;;;;;;;GAWG;AACH,wBAAgB,iBAAiB,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,CASzD;AAID;;;;;;;;;;GAUG;AACH,wBAAgB,YAAY,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,EAAE,CA8BtD;AAID,kDAAkD;AAClD,wBAAgB,MAAM,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,CAE9C;AAsJD;;;;;;;;;;;;;;;;GAgBG;AACH,wBAAsB,kBAAkB,CAAC,IAAI,EAAE,iBAAiB,GAAG,OAAO,CAAC,aAAa,CAAC,CA6FxF;AAID;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,wBAAsB,YAAY,CAAC,IAAI,EAAE,mBAAmB,GAAG,OAAO,CAAC,kBAAkB,CAAC,CAqFzF"}