@soundbi/sound-connect 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +111 -0
- package/dist/__tests__/ingest.test.d.ts +18 -0
- package/dist/__tests__/ingest.test.d.ts.map +1 -0
- package/dist/__tests__/ingest.test.js +639 -0
- package/dist/__tests__/ingest.test.js.map +1 -0
- package/dist/__tests__/isolation.test.d.ts +12 -0
- package/dist/__tests__/isolation.test.d.ts.map +1 -0
- package/dist/__tests__/isolation.test.js +149 -0
- package/dist/__tests__/isolation.test.js.map +1 -0
- package/dist/__tests__/retry-queue.test.d.ts +11 -0
- package/dist/__tests__/retry-queue.test.d.ts.map +1 -0
- package/dist/__tests__/retry-queue.test.js +458 -0
- package/dist/__tests__/retry-queue.test.js.map +1 -0
- package/dist/auth.d.ts +80 -0
- package/dist/auth.d.ts.map +1 -0
- package/dist/auth.js +211 -0
- package/dist/auth.js.map +1 -0
- package/dist/config.d.ts +35 -0
- package/dist/config.d.ts.map +1 -0
- package/dist/config.js +66 -0
- package/dist/config.js.map +1 -0
- package/dist/index.d.ts +23 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +100 -0
- package/dist/index.js.map +1 -0
- package/dist/ingest.d.ts +253 -0
- package/dist/ingest.d.ts.map +1 -0
- package/dist/ingest.js +573 -0
- package/dist/ingest.js.map +1 -0
- package/dist/proxy.d.ts +79 -0
- package/dist/proxy.d.ts.map +1 -0
- package/dist/proxy.js +217 -0
- package/dist/proxy.js.map +1 -0
- package/dist/retry-queue.d.ts +236 -0
- package/dist/retry-queue.d.ts.map +1 -0
- package/dist/retry-queue.js +461 -0
- package/dist/retry-queue.js.map +1 -0
- package/dist/tools.d.ts +75 -0
- package/dist/tools.d.ts.map +1 -0
- package/dist/tools.js +368 -0
- package/dist/tools.js.map +1 -0
- package/package.json +36 -0
package/dist/ingest.js
ADDED
|
@@ -0,0 +1,573 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* File ingestion logic for the Sound Connect bridge (STORY-011, STORY-012, STORY-013).
|
|
3
|
+
*
|
|
4
|
+
* STORY-011 implements:
|
|
5
|
+
* - Path validation and confinement (no directory traversal — AC3)
|
|
6
|
+
* - Markdown file reading and normalization (AC1)
|
|
7
|
+
* - Content chunking for large files (AC1)
|
|
8
|
+
* - SHA-256 content hash for idempotency (AC2, ADR-004)
|
|
9
|
+
* - Provenance attachment and POST to /ingest/:slug (AC2)
|
|
10
|
+
* - Summary return: chunks ingested, deduped count (AC4)
|
|
11
|
+
*
|
|
12
|
+
* STORY-012 extends to:
|
|
13
|
+
* - Folder enumeration supporting .md and transcript text files (.txt/.vtt/.srt)
|
|
14
|
+
* - Per-file source_type reflecting the file kind (markdown vs transcript)
|
|
15
|
+
* - Idempotency across folder re-runs (sha256 already present → deduped)
|
|
16
|
+
* - Per-file result table (ingested / deduped / failed)
|
|
17
|
+
*
|
|
18
|
+
* STORY-013 extends to:
|
|
19
|
+
* - When `queueOnFailure` is enabled, network errors and 5xx responses write the
|
|
20
|
+
* chunk to the local retry queue instead of throwing (ADR-011).
|
|
21
|
+
* - 4xx responses still throw immediately (permanent auth/access error — queue won't help).
|
|
22
|
+
* - IngestSummary gains a `chunks_queued` field reporting items sent to the retry queue.
|
|
23
|
+
*
|
|
24
|
+
* ADR-006: v1 handles text/markdown and transcript plaintext only. Binary formats
|
|
25
|
+
* (PDF, .xlsx, .docx) deferred to v1.1.
|
|
26
|
+
* ADR-004: sha256 of normalized content is the idempotency key.
|
|
27
|
+
* ADR-011: All errors are thrown with descriptive messages; callers surface them
|
|
28
|
+
* as MCP tool errors — never silent failures.
|
|
29
|
+
*/
|
|
30
|
+
import { createHash } from 'node:crypto';
|
|
31
|
+
import { readFile, stat, readdir } from 'node:fs/promises';
|
|
32
|
+
import { resolve, extname, basename, join } from 'node:path';
|
|
33
|
+
import { enqueueFailedChunk } from './retry-queue.js';
|
|
34
|
+
// ── Constants ──────────────────────────────────────────────────────────────────
|
|
35
|
+
/**
|
|
36
|
+
* Maximum single-chunk size in characters.
|
|
37
|
+
* Files larger than this are split into multiple chunks (AC1: chunk if large).
|
|
38
|
+
* ~6 000 chars ≈ ~1 500 tokens — a reasonable semantic unit for retrieval.
|
|
39
|
+
*/
|
|
40
|
+
const CHUNK_SIZE_CHARS = 6_000;
|
|
41
|
+
/**
|
|
42
|
+
* Overlap between consecutive chunks in characters.
|
|
43
|
+
* Preserves context across chunk boundaries for better retrieval quality.
|
|
44
|
+
*/
|
|
45
|
+
const CHUNK_OVERLAP_CHARS = 200;
|
|
46
|
+
/**
|
|
47
|
+
* Maximum file size accepted for ingestion (10 MB — ADR-004 body limit).
|
|
48
|
+
* The backend enforces this at the HTTP layer; we mirror it locally for a
|
|
49
|
+
* clear early error before the network round-trip.
|
|
50
|
+
*/
|
|
51
|
+
const MAX_FILE_BYTES = 10 * 1024 * 1024;
|
|
52
|
+
/**
|
|
53
|
+
* File extensions accepted for ingestion (ADR-006: text/markdown + transcript plaintext).
|
|
54
|
+
* Binary formats (PDF, .xlsx, .docx) are deferred to v1.1.
|
|
55
|
+
*/
|
|
56
|
+
export const SUPPORTED_EXTENSIONS = new Set(['.md', '.txt', '.vtt', '.srt']);
|
|
57
|
+
/**
|
|
58
|
+
* Derive the source_type provenance value from a file extension (STORY-012, ADR-004).
|
|
59
|
+
* Returns 'markdown' for .md files, 'transcript' for .txt/.vtt/.srt.
|
|
60
|
+
*/
|
|
61
|
+
export function sourceTypeForExt(ext) {
|
|
62
|
+
return ext === '.md' ? 'markdown' : 'transcript';
|
|
63
|
+
}
|
|
64
|
+
// ── Path validation (AC3, STORY-011/STORY-012) ───────────────────────────────
|
|
65
|
+
/**
|
|
66
|
+
* Resolves and validates a single ingestable file path.
|
|
67
|
+
*
|
|
68
|
+
* Rules enforced:
|
|
69
|
+
* 1. Path must not be empty.
|
|
70
|
+
* 2. File must exist and be a regular file (not a directory or symlink loop).
|
|
71
|
+
* 3. File extension must be in SUPPORTED_EXTENSIONS (ADR-006: .md/.txt/.vtt/.srt).
|
|
72
|
+
* 4. File size must not exceed MAX_FILE_BYTES (ADR-004 body limit).
|
|
73
|
+
*
|
|
74
|
+
* Traversal confinement: `resolve()` normalises `../` sequences so the resolved
|
|
75
|
+
* path is always absolute. We do NOT restrict to a whitelist directory because
|
|
76
|
+
* peers legitimately have notes and transcripts anywhere on their machine. We do
|
|
77
|
+
* block the obvious attack vectors: empty paths, non-files, unsupported extensions.
|
|
78
|
+
*
|
|
79
|
+
* @param rawPath The path supplied by the tool caller.
|
|
80
|
+
* @returns Resolved absolute path.
|
|
81
|
+
* @throws Error with a descriptive message on any validation failure.
|
|
82
|
+
*/
|
|
83
|
+
export async function validateAndResolvePath(rawPath) {
|
|
84
|
+
if (!rawPath || rawPath.trim() === '') {
|
|
85
|
+
throw new Error('Path must not be empty.');
|
|
86
|
+
}
|
|
87
|
+
// Normalise: resolve relative paths against cwd, collapse ../ sequences.
|
|
88
|
+
const resolved = resolve(rawPath.trim());
|
|
89
|
+
// Check for null bytes (path injection attempt).
|
|
90
|
+
if (resolved.includes('\0')) {
|
|
91
|
+
throw new Error(`Invalid path: contains null byte.`);
|
|
92
|
+
}
|
|
93
|
+
// Must be a supported text format (ADR-006: markdown + transcripts in v1).
|
|
94
|
+
const ext = extname(resolved).toLowerCase();
|
|
95
|
+
if (!SUPPORTED_EXTENSIONS.has(ext)) {
|
|
96
|
+
throw new Error(`File "${basename(resolved)}" has extension "${ext || '(none)'}". ` +
|
|
97
|
+
'Only markdown (.md) and transcript (.txt, .vtt, .srt) files are supported in v1. ' +
|
|
98
|
+
'Binary format support (PDF, .docx, .xlsx) is planned for v1.1.');
|
|
99
|
+
}
|
|
100
|
+
// Verify the file exists and is a regular file.
|
|
101
|
+
let fileSize;
|
|
102
|
+
try {
|
|
103
|
+
const info = await stat(resolved);
|
|
104
|
+
if (!info.isFile()) {
|
|
105
|
+
throw new Error(`Path "${resolved}" is not a regular file.`);
|
|
106
|
+
}
|
|
107
|
+
fileSize = info.size;
|
|
108
|
+
}
|
|
109
|
+
catch (err) {
|
|
110
|
+
if (err.code === 'ENOENT') {
|
|
111
|
+
throw new Error(`File not found: "${resolved}". Check the path and try again.`);
|
|
112
|
+
}
|
|
113
|
+
if (err.code === 'EACCES') {
|
|
114
|
+
throw new Error(`Permission denied reading "${resolved}".`);
|
|
115
|
+
}
|
|
116
|
+
throw err;
|
|
117
|
+
}
|
|
118
|
+
// Size guard — mirror the backend's 10 MB body limit (ADR-004).
|
|
119
|
+
if (fileSize > MAX_FILE_BYTES) {
|
|
120
|
+
throw new Error(`File "${basename(resolved)}" is ${(fileSize / 1024 / 1024).toFixed(1)} MB, ` +
|
|
121
|
+
`which exceeds the 10 MB ingestion limit.`);
|
|
122
|
+
}
|
|
123
|
+
return resolved;
|
|
124
|
+
}
|
|
125
|
+
// ── Folder enumeration (STORY-012, AC1) ──────────────────────────────────────
|
|
126
|
+
/**
|
|
127
|
+
* Parse the `glob` argument from ingest_folder into a Set of allowed extensions.
|
|
128
|
+
*
|
|
129
|
+
* Accepts patterns like "*.md", "*.vtt", "*.txt", "*.srt".
|
|
130
|
+
* If the pattern matches a known SUPPORTED_EXTENSIONS entry, only that extension
|
|
131
|
+
* is returned. If glob is undefined/empty, all SUPPORTED_EXTENSIONS are returned.
|
|
132
|
+
* If glob specifies an unsupported extension, throws with a clear message.
|
|
133
|
+
*
|
|
134
|
+
* This is intentionally simple — STORY-012 AC1 says "glob?" but the only
|
|
135
|
+
* meaningful variation is filtering by extension, not full glob matching.
|
|
136
|
+
* Full glob support (**, recursive patterns) is deferred to v1.1.
|
|
137
|
+
*/
|
|
138
|
+
export function parseGlobToExtensions(glob) {
|
|
139
|
+
if (!glob || glob.trim() === '') {
|
|
140
|
+
return new Set(SUPPORTED_EXTENSIONS);
|
|
141
|
+
}
|
|
142
|
+
// Extract the extension from a pattern like "*.md", "*.vtt", "docs/*.txt".
|
|
143
|
+
const match = /\*(\.[a-z0-9]+)$/i.exec(glob.trim());
|
|
144
|
+
if (!match) {
|
|
145
|
+
// No wildcard extension found — treat as all supported extensions.
|
|
146
|
+
return new Set(SUPPORTED_EXTENSIONS);
|
|
147
|
+
}
|
|
148
|
+
const ext = match[1].toLowerCase();
|
|
149
|
+
if (!SUPPORTED_EXTENSIONS.has(ext)) {
|
|
150
|
+
throw new Error(`glob pattern "${glob}" requests extension "${ext}" which is not supported. ` +
|
|
151
|
+
`Supported extensions: ${[...SUPPORTED_EXTENSIONS].join(', ')}. ` +
|
|
152
|
+
'Binary formats (PDF, .docx, .xlsx) are planned for v1.1.');
|
|
153
|
+
}
|
|
154
|
+
return new Set([ext]);
|
|
155
|
+
}
|
|
156
|
+
/**
|
|
157
|
+
* Enumerate all ingestable files in a folder (non-recursive, flat listing).
|
|
158
|
+
*
|
|
159
|
+
* Returns absolute paths of files whose extension is in the allowed set.
|
|
160
|
+
* Silently skips subdirectories and files with unsupported extensions.
|
|
161
|
+
* Throws if the folder does not exist or is not a directory.
|
|
162
|
+
*
|
|
163
|
+
* STORY-012 AC1: enumerates .md and transcript text files (.txt/.vtt/.srt).
|
|
164
|
+
* Recursion is not in scope for v1 — flat directory only.
|
|
165
|
+
*
|
|
166
|
+
* @param folderPath Absolute path to the folder to enumerate.
|
|
167
|
+
* @param allowedExts Set of lowercase extensions to include (from parseGlobToExtensions).
|
|
168
|
+
* @returns Sorted list of absolute file paths.
|
|
169
|
+
*/
|
|
170
|
+
export async function enumerateFolder(folderPath, allowedExts) {
|
|
171
|
+
let entries;
|
|
172
|
+
try {
|
|
173
|
+
entries = await readdir(folderPath, { withFileTypes: true });
|
|
174
|
+
}
|
|
175
|
+
catch (err) {
|
|
176
|
+
if (err.code === 'ENOENT') {
|
|
177
|
+
throw new Error(`Folder not found: "${folderPath}". Check the path and try again.`);
|
|
178
|
+
}
|
|
179
|
+
if (err.code === 'ENOTDIR') {
|
|
180
|
+
throw new Error(`Path "${folderPath}" is not a directory.`);
|
|
181
|
+
}
|
|
182
|
+
if (err.code === 'EACCES') {
|
|
183
|
+
throw new Error(`Permission denied reading folder "${folderPath}".`);
|
|
184
|
+
}
|
|
185
|
+
throw err;
|
|
186
|
+
}
|
|
187
|
+
const files = [];
|
|
188
|
+
for (const entry of entries) {
|
|
189
|
+
if (!entry.isFile())
|
|
190
|
+
continue;
|
|
191
|
+
const ext = extname(entry.name).toLowerCase();
|
|
192
|
+
if (!allowedExts.has(ext))
|
|
193
|
+
continue;
|
|
194
|
+
files.push(join(folderPath, entry.name));
|
|
195
|
+
}
|
|
196
|
+
// Deterministic ordering so result tables are consistent across runs.
|
|
197
|
+
files.sort();
|
|
198
|
+
return files;
|
|
199
|
+
}
|
|
200
|
+
// ── Normalization (AC1) ────────────────────────────────────────────────────────
|
|
201
|
+
/**
|
|
202
|
+
* Normalize markdown content before hashing and chunking.
|
|
203
|
+
*
|
|
204
|
+
* Normalization steps:
|
|
205
|
+
* 1. Normalize line endings to LF.
|
|
206
|
+
* 2. Strip trailing whitespace from each line.
|
|
207
|
+
* 3. Collapse runs of 3+ blank lines to 2 blank lines (preserve paragraph spacing).
|
|
208
|
+
* 4. Trim leading/trailing blank lines from the document.
|
|
209
|
+
*
|
|
210
|
+
* This ensures that whitespace-only edits do not produce new sha256 hashes
|
|
211
|
+
* (no spurious re-ingestion) while preserving meaningful structure.
|
|
212
|
+
*/
|
|
213
|
+
export function normalizeMarkdown(content) {
|
|
214
|
+
return content
|
|
215
|
+
.replace(/\r\n/g, '\n') // CRLF → LF
|
|
216
|
+
.replace(/\r/g, '\n') // bare CR → LF
|
|
217
|
+
.split('\n')
|
|
218
|
+
.map(line => line.trimEnd()) // strip trailing whitespace per line
|
|
219
|
+
.join('\n')
|
|
220
|
+
.replace(/\n{3,}/g, '\n\n') // collapse 3+ blank lines to 2
|
|
221
|
+
.trim();
|
|
222
|
+
}
|
|
223
|
+
// ── Chunking (AC1) ────────────────────────────────────────────────────────────
|
|
224
|
+
/**
|
|
225
|
+
* Split normalized content into overlapping chunks for retrieval-friendly storage.
|
|
226
|
+
*
|
|
227
|
+
* Strategy: paragraph-aware sliding window.
|
|
228
|
+
* - Split on double-newlines (paragraph boundaries) to avoid cutting mid-sentence.
|
|
229
|
+
* - Accumulate paragraphs until the chunk would exceed CHUNK_SIZE_CHARS.
|
|
230
|
+
* - Start the next chunk with the last CHUNK_OVERLAP_CHARS of the previous chunk
|
|
231
|
+
* (approximate — overlap at paragraph boundary nearest to the overlap target).
|
|
232
|
+
*
|
|
233
|
+
* If the entire content fits in one chunk, returns a single-element array.
|
|
234
|
+
*/
|
|
235
|
+
export function chunkContent(content) {
|
|
236
|
+
if (content.length <= CHUNK_SIZE_CHARS) {
|
|
237
|
+
return [content];
|
|
238
|
+
}
|
|
239
|
+
const paragraphs = content.split(/\n\n+/);
|
|
240
|
+
const chunks = [];
|
|
241
|
+
let current = '';
|
|
242
|
+
for (const paragraph of paragraphs) {
|
|
243
|
+
const candidate = current ? `${current}\n\n${paragraph}` : paragraph;
|
|
244
|
+
if (candidate.length > CHUNK_SIZE_CHARS && current) {
|
|
245
|
+
// Flush current chunk.
|
|
246
|
+
chunks.push(current);
|
|
247
|
+
// Start next chunk with overlap: take the tail of the current chunk.
|
|
248
|
+
const overlapStart = Math.max(0, current.length - CHUNK_OVERLAP_CHARS);
|
|
249
|
+
const overlap = current.slice(overlapStart);
|
|
250
|
+
current = `${overlap}\n\n${paragraph}`;
|
|
251
|
+
}
|
|
252
|
+
else {
|
|
253
|
+
current = candidate;
|
|
254
|
+
}
|
|
255
|
+
}
|
|
256
|
+
if (current) {
|
|
257
|
+
chunks.push(current);
|
|
258
|
+
}
|
|
259
|
+
return chunks.length > 0 ? chunks : [content];
|
|
260
|
+
}
|
|
261
|
+
// ── Hashing (AC2, ADR-004) ────────────────────────────────────────────────────
|
|
262
|
+
/** Compute the SHA-256 hex digest of a string. */
|
|
263
|
+
export function sha256(content) {
|
|
264
|
+
return createHash('sha256').update(content, 'utf8').digest('hex');
|
|
265
|
+
}
|
|
266
|
+
/**
|
|
267
|
+
* Attempt a single POST to /ingest/:slug and classify the outcome.
|
|
268
|
+
*
|
|
269
|
+
* Does NOT throw — returns a typed outcome so the caller decides whether
|
|
270
|
+
* to queue, throw, or count as deduped.
|
|
271
|
+
*
|
|
272
|
+
* Classification:
|
|
273
|
+
* transient — network error or 5xx: idempotency (ADR-004) makes retry safe.
|
|
274
|
+
* permanent — 4xx: auth/access error that won't resolve by retrying.
|
|
275
|
+
* success — 2xx.
|
|
276
|
+
*
|
|
277
|
+
* Note: 413 (payload too large) is treated as permanent — retrying the same
|
|
278
|
+
* oversized chunk will never succeed.
|
|
279
|
+
*/
|
|
280
|
+
async function tryPostChunk(backendUrl, clientSlug, token, payload) {
|
|
281
|
+
const url = `${backendUrl}/ingest/${encodeURIComponent(clientSlug)}`;
|
|
282
|
+
let res;
|
|
283
|
+
try {
|
|
284
|
+
res = await fetch(url, {
|
|
285
|
+
method: 'POST',
|
|
286
|
+
headers: {
|
|
287
|
+
'Content-Type': 'application/json',
|
|
288
|
+
'Authorization': `Bearer ${token}`,
|
|
289
|
+
},
|
|
290
|
+
body: JSON.stringify(payload),
|
|
291
|
+
});
|
|
292
|
+
}
|
|
293
|
+
catch (err) {
|
|
294
|
+
// Network-level failure (DNS, timeout, ECONNREFUSED) — transient.
|
|
295
|
+
return {
|
|
296
|
+
kind: 'transient',
|
|
297
|
+
error: `Backend unreachable at ${url}: ${err.message}. ` +
|
|
298
|
+
'Check your network connection and try again.',
|
|
299
|
+
};
|
|
300
|
+
}
|
|
301
|
+
let bodyText = '';
|
|
302
|
+
try {
|
|
303
|
+
bodyText = await res.text();
|
|
304
|
+
}
|
|
305
|
+
catch { /* ignore */ }
|
|
306
|
+
// 413 = payload too large (permanent — retrying the same chunk won't help).
|
|
307
|
+
if (res.status === 413) {
|
|
308
|
+
return {
|
|
309
|
+
kind: 'permanent',
|
|
310
|
+
error: 'Backend rejected the payload as too large (413). The file or chunk exceeds the 10 MB limit.',
|
|
311
|
+
};
|
|
312
|
+
}
|
|
313
|
+
// Any other 4xx = permanent (auth / access).
|
|
314
|
+
if (res.status >= 400 && res.status < 500) {
|
|
315
|
+
let hint = '';
|
|
316
|
+
try {
|
|
317
|
+
const parsed = JSON.parse(bodyText);
|
|
318
|
+
hint = (parsed['hint'] ?? parsed['error'] ?? parsed['message'] ?? '');
|
|
319
|
+
}
|
|
320
|
+
catch { /* not JSON */ }
|
|
321
|
+
if (res.status === 401) {
|
|
322
|
+
return {
|
|
323
|
+
kind: 'permanent',
|
|
324
|
+
error: 'Bearer token rejected by backend (401). Run `npx @soundbi/sound-connect login` to re-authenticate.',
|
|
325
|
+
};
|
|
326
|
+
}
|
|
327
|
+
if (res.status === 403) {
|
|
328
|
+
return {
|
|
329
|
+
kind: 'permanent',
|
|
330
|
+
error: `Access denied to client "${clientSlug}" (403). ${hint || 'Verify your Sound Connect membership.'}`,
|
|
331
|
+
};
|
|
332
|
+
}
|
|
333
|
+
return {
|
|
334
|
+
kind: 'permanent',
|
|
335
|
+
error: `Backend returned HTTP ${res.status}${hint ? `: ${hint}` : ''}. ${bodyText.slice(0, 200)}`,
|
|
336
|
+
};
|
|
337
|
+
}
|
|
338
|
+
// 5xx or other non-2xx — transient.
|
|
339
|
+
if (!res.ok) {
|
|
340
|
+
return {
|
|
341
|
+
kind: 'transient',
|
|
342
|
+
error: `Backend returned HTTP ${res.status}. ${bodyText.slice(0, 200)}`,
|
|
343
|
+
};
|
|
344
|
+
}
|
|
345
|
+
// 2xx success.
|
|
346
|
+
try {
|
|
347
|
+
return { kind: 'success', response: JSON.parse(bodyText) };
|
|
348
|
+
}
|
|
349
|
+
catch {
|
|
350
|
+
// 200 with non-JSON body — treat as success, non-deduped.
|
|
351
|
+
return { kind: 'success', response: { ok: true, deduped: false } };
|
|
352
|
+
}
|
|
353
|
+
}
|
|
354
|
+
/**
|
|
355
|
+
* POST a single chunk to the backend /ingest/:slug endpoint.
|
|
356
|
+
*
|
|
357
|
+
* Returns { deduped: true } when the backend reports the sha256 already exists.
|
|
358
|
+
* Throws on network errors or non-2xx responses (ADR-011: fail closed).
|
|
359
|
+
*
|
|
360
|
+
* This wrapper is kept for backward compatibility with existing callers that
|
|
361
|
+
* expect the original throw-on-failure contract (STORY-011/012 tests).
|
|
362
|
+
*/
|
|
363
|
+
async function postChunk(backendUrl, clientSlug, token, payload) {
|
|
364
|
+
const outcome = await tryPostChunk(backendUrl, clientSlug, token, payload);
|
|
365
|
+
if (outcome.kind === 'success')
|
|
366
|
+
return outcome.response;
|
|
367
|
+
throw new Error(outcome.error);
|
|
368
|
+
}
|
|
369
|
+
// ── Public API (AC1–AC4) ──────────────────────────────────────────────────────
|
|
370
|
+
/**
|
|
371
|
+
* Ingest a single local text file (markdown or transcript) into the Sound Connect corpus.
|
|
372
|
+
*
|
|
373
|
+
* Full pipeline:
|
|
374
|
+
* 1. Validate path — supports .md, .txt, .vtt, .srt (STORY-012 AC1, ADR-006).
|
|
375
|
+
* 2. Read file.
|
|
376
|
+
* 3. Normalize content (line endings, trailing whitespace, blank lines).
|
|
377
|
+
* 4. Chunk if large.
|
|
378
|
+
* 5. Hash each chunk (ADR-004 idempotency key).
|
|
379
|
+
* 6. Attach provenance with source_type derived from extension (STORY-012 AC2).
|
|
380
|
+
* 7. POST each chunk to /ingest/:slug.
|
|
381
|
+
* STORY-013: When queueOnFailure=true, network errors and 5xx responses write
|
|
382
|
+
* the chunk to the local retry queue instead of throwing. 4xx errors still throw.
|
|
383
|
+
* 8. Return summary (includes chunks_queued when queueOnFailure is used).
|
|
384
|
+
*
|
|
385
|
+
* @throws Error with a descriptive message on any failure (ADR-011).
|
|
386
|
+
*/
|
|
387
|
+
export async function ingestMarkdownFile(opts) {
|
|
388
|
+
const { filePath, backendUrl, clientSlug, token, workstreamSlug, authorEmail, queueOnFailure } = opts;
|
|
389
|
+
// Validate and resolve path (supports .md + transcripts).
|
|
390
|
+
const resolvedPath = await validateAndResolvePath(filePath);
|
|
391
|
+
const filename = basename(resolvedPath);
|
|
392
|
+
// Derive source_type from extension (STORY-012 AC2, ADR-004 invariant 2).
|
|
393
|
+
const ext = extname(resolvedPath).toLowerCase();
|
|
394
|
+
const sourceType = sourceTypeForExt(ext);
|
|
395
|
+
// Read file.
|
|
396
|
+
const rawContent = await readFile(resolvedPath, 'utf8');
|
|
397
|
+
// Normalize (shared for markdown and transcript plaintext).
|
|
398
|
+
const normalized = normalizeMarkdown(rawContent);
|
|
399
|
+
if (!normalized) {
|
|
400
|
+
throw new Error(`File "${filename}" is empty after normalization. Nothing to ingest.`);
|
|
401
|
+
}
|
|
402
|
+
// Full-content hash for the summary (hash of the complete normalized document).
|
|
403
|
+
const contentHash = sha256(normalized);
|
|
404
|
+
// Chunk.
|
|
405
|
+
const chunks = chunkContent(normalized);
|
|
406
|
+
// Provenance — author_email from token claim or config (ADR-004 invariant 2).
|
|
407
|
+
const effectiveAuthorEmail = authorEmail ?? '';
|
|
408
|
+
const timestamp = new Date().toISOString();
|
|
409
|
+
let chunksIngested = 0;
|
|
410
|
+
let chunksDeduped = 0;
|
|
411
|
+
let chunksQueued = 0;
|
|
412
|
+
// POST each chunk with provenance.
|
|
413
|
+
for (let i = 0; i < chunks.length; i++) {
|
|
414
|
+
const chunkText = chunks[i];
|
|
415
|
+
const chunkHash = chunks.length === 1 ? contentHash : sha256(chunkText);
|
|
416
|
+
// Filename disambiguated per-chunk when there are multiple chunks.
|
|
417
|
+
const chunkFilename = chunks.length === 1
|
|
418
|
+
? filename
|
|
419
|
+
: `${filename}#chunk-${i + 1}-of-${chunks.length}`;
|
|
420
|
+
const payload = {
|
|
421
|
+
source_type: sourceType,
|
|
422
|
+
filename: chunkFilename,
|
|
423
|
+
content: chunkText,
|
|
424
|
+
sha256: chunkHash,
|
|
425
|
+
timestamp,
|
|
426
|
+
author_email: effectiveAuthorEmail,
|
|
427
|
+
...(workstreamSlug ? { workstream_slug: workstreamSlug } : {}),
|
|
428
|
+
};
|
|
429
|
+
if (queueOnFailure) {
|
|
430
|
+
// STORY-013: use tryPostChunk so we can distinguish transient vs permanent failures.
|
|
431
|
+
const outcome = await tryPostChunk(backendUrl, clientSlug, token, payload);
|
|
432
|
+
if (outcome.kind === 'success') {
|
|
433
|
+
if (outcome.response.deduped) {
|
|
434
|
+
chunksDeduped++;
|
|
435
|
+
}
|
|
436
|
+
else {
|
|
437
|
+
chunksIngested++;
|
|
438
|
+
}
|
|
439
|
+
}
|
|
440
|
+
else if (outcome.kind === 'transient') {
|
|
441
|
+
// Transient (network / 5xx) — write to retry queue. ADR-004 idempotency makes re-send safe.
|
|
442
|
+
await enqueueFailedChunk(backendUrl, clientSlug, payload, outcome.error);
|
|
443
|
+
chunksQueued++;
|
|
444
|
+
}
|
|
445
|
+
else {
|
|
446
|
+
// Permanent (4xx) — throw immediately. Queueing won't help (auth/access error).
|
|
447
|
+
throw new Error(outcome.error);
|
|
448
|
+
}
|
|
449
|
+
}
|
|
450
|
+
else {
|
|
451
|
+
// Original behaviour: throw on any failure.
|
|
452
|
+
const result = await postChunk(backendUrl, clientSlug, token, payload);
|
|
453
|
+
if (result.deduped) {
|
|
454
|
+
chunksDeduped++;
|
|
455
|
+
}
|
|
456
|
+
else {
|
|
457
|
+
chunksIngested++;
|
|
458
|
+
}
|
|
459
|
+
}
|
|
460
|
+
}
|
|
461
|
+
// Return summary.
|
|
462
|
+
return {
|
|
463
|
+
chunks_sent: chunks.length,
|
|
464
|
+
chunks_ingested: chunksIngested,
|
|
465
|
+
chunks_deduped: chunksDeduped,
|
|
466
|
+
chunks_queued: chunksQueued,
|
|
467
|
+
file: resolvedPath,
|
|
468
|
+
content_hash: contentHash,
|
|
469
|
+
};
|
|
470
|
+
}
|
|
471
|
+
// ── ingestFolder (STORY-012) ──────────────────────────────────────────────────
|
|
472
|
+
/**
|
|
473
|
+
* Bulk-ingest a directory of markdown and transcript text files.
|
|
474
|
+
*
|
|
475
|
+
* Pipeline (STORY-012):
|
|
476
|
+
* 1. Resolve and validate the folder path.
|
|
477
|
+
* 2. Parse glob to determine which extensions to include (AC1).
|
|
478
|
+
* 3. Enumerate matching files in the folder (flat, non-recursive).
|
|
479
|
+
* 4. For each file: run through the ingestMarkdownFile pipeline (STORY-011 path).
|
|
480
|
+
* - deduped = all chunks were already present (idempotency, AC3).
|
|
481
|
+
* - ingested = at least one new chunk was accepted.
|
|
482
|
+
* - failed = any error during read/post.
|
|
483
|
+
* 5. Return per-file result table + folder-level totals (AC4).
|
|
484
|
+
*
|
|
485
|
+
* Idempotency (AC3): re-running the folder produces files_ingested=0 and
|
|
486
|
+
* files_deduped=N (one per file) because sha256 keys are already present.
|
|
487
|
+
*
|
|
488
|
+
* ADR-011: per-file failures are captured in the result table and do NOT abort
|
|
489
|
+
* the rest of the run — callers see which files failed and why.
|
|
490
|
+
*
|
|
491
|
+
* @throws Error only if the folder itself cannot be accessed (not per-file errors).
|
|
492
|
+
*/
|
|
493
|
+
export async function ingestFolder(opts) {
|
|
494
|
+
const { folderPath, glob, backendUrl, clientSlug, token, workstreamSlug, authorEmail } = opts;
|
|
495
|
+
// Resolve and validate folder path.
|
|
496
|
+
const resolvedFolder = resolve(folderPath.trim());
|
|
497
|
+
// Verify the folder exists and is a directory.
|
|
498
|
+
try {
|
|
499
|
+
const info = await stat(resolvedFolder);
|
|
500
|
+
if (!info.isDirectory()) {
|
|
501
|
+
throw new Error(`Path "${resolvedFolder}" is not a directory.`);
|
|
502
|
+
}
|
|
503
|
+
}
|
|
504
|
+
catch (err) {
|
|
505
|
+
if (err.code === 'ENOENT') {
|
|
506
|
+
throw new Error(`Folder not found: "${resolvedFolder}". Check the path and try again.`);
|
|
507
|
+
}
|
|
508
|
+
if (err.code === 'EACCES') {
|
|
509
|
+
throw new Error(`Permission denied reading folder "${resolvedFolder}".`);
|
|
510
|
+
}
|
|
511
|
+
throw err;
|
|
512
|
+
}
|
|
513
|
+
// Parse glob into allowed extensions (AC1).
|
|
514
|
+
const allowedExts = parseGlobToExtensions(glob);
|
|
515
|
+
// Enumerate matching files (AC1).
|
|
516
|
+
const files = await enumerateFolder(resolvedFolder, allowedExts);
|
|
517
|
+
const results = [];
|
|
518
|
+
let filesIngested = 0;
|
|
519
|
+
let filesDeduped = 0;
|
|
520
|
+
let filesFailed = 0;
|
|
521
|
+
// Process each file through the STORY-011 ingest path (AC2).
|
|
522
|
+
for (const filePath of files) {
|
|
523
|
+
try {
|
|
524
|
+
const summary = await ingestMarkdownFile({
|
|
525
|
+
filePath,
|
|
526
|
+
backendUrl,
|
|
527
|
+
clientSlug,
|
|
528
|
+
token,
|
|
529
|
+
workstreamSlug,
|
|
530
|
+
authorEmail,
|
|
531
|
+
});
|
|
532
|
+
// AC3: Idempotency — fully deduped means all chunks were already present.
|
|
533
|
+
const fullyDeduped = summary.chunks_ingested === 0 && summary.chunks_deduped > 0;
|
|
534
|
+
if (fullyDeduped) {
|
|
535
|
+
filesDeduped++;
|
|
536
|
+
results.push({
|
|
537
|
+
file: filePath,
|
|
538
|
+
status: 'deduped',
|
|
539
|
+
chunks_sent: summary.chunks_sent,
|
|
540
|
+
chunks_deduped: summary.chunks_deduped,
|
|
541
|
+
});
|
|
542
|
+
}
|
|
543
|
+
else {
|
|
544
|
+
filesIngested++;
|
|
545
|
+
results.push({
|
|
546
|
+
file: filePath,
|
|
547
|
+
status: 'ingested',
|
|
548
|
+
chunks_sent: summary.chunks_sent,
|
|
549
|
+
chunks_ingested: summary.chunks_ingested,
|
|
550
|
+
chunks_deduped: summary.chunks_deduped,
|
|
551
|
+
});
|
|
552
|
+
}
|
|
553
|
+
}
|
|
554
|
+
catch (err) {
|
|
555
|
+
// ADR-011: per-file failures captured, do not abort the folder run.
|
|
556
|
+
filesFailed++;
|
|
557
|
+
results.push({
|
|
558
|
+
file: filePath,
|
|
559
|
+
status: 'failed',
|
|
560
|
+
error: (err instanceof Error) ? err.message : String(err),
|
|
561
|
+
});
|
|
562
|
+
}
|
|
563
|
+
}
|
|
564
|
+
return {
|
|
565
|
+
folder: resolvedFolder,
|
|
566
|
+
files_found: files.length,
|
|
567
|
+
files_ingested: filesIngested,
|
|
568
|
+
files_deduped: filesDeduped,
|
|
569
|
+
files_failed: filesFailed,
|
|
570
|
+
results,
|
|
571
|
+
};
|
|
572
|
+
}
|
|
573
|
+
//# sourceMappingURL=ingest.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"ingest.js","sourceRoot":"","sources":["../src/ingest.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,kBAAkB,CAAC;AAE3D,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAC7D,OAAO,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAC;AAEtD,kFAAkF;AAElF;;;;GAIG;AACH,MAAM,gBAAgB,GAAG,KAAK,CAAC;AAE/B;;;GAGG;AACH,MAAM,mBAAmB,GAAG,GAAG,CAAC;AAEhC;;;;GAIG;AACH,MAAM,cAAc,GAAG,EAAE,GAAG,IAAI,GAAG,IAAI,CAAC;AAExC;;;GAGG;AACH,MAAM,CAAC,MAAM,oBAAoB,GAAG,IAAI,GAAG,CAAC,CAAC,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;AAE7E;;;GAGG;AACH,MAAM,UAAU,gBAAgB,CAAC,GAAW;IAC1C,OAAO,GAAG,KAAK,KAAK,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,YAAY,CAAC;AACnD,CAAC;AA+GD,gFAAgF;AAEhF;;;;;;;;;;;;;;;;;GAiBG;AACH,MAAM,CAAC,KAAK,UAAU,sBAAsB,CAAC,OAAe;IAC1D,IAAI,CAAC,OAAO,IAAI,OAAO,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC;QACtC,MAAM,IAAI,KAAK,CAAC,yBAAyB,CAAC,CAAC;IAC7C,CAAC;IAED,yEAAyE;IACzE,MAAM,QAAQ,GAAG,OAAO,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC;IAEzC,iDAAiD;IACjD,IAAI,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;QAC5B,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;IACvD,CAAC;IAED,2EAA2E;IAC3E,MAAM,GAAG,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAC,WAAW,EAAE,CAAC;IAC5C,IAAI,CAAC,oBAAoB,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;QACnC,MAAM,IAAI,KAAK,CACb,SAAS,QAAQ,CAAC,QAAQ,CAAC,oBAAoB,GAAG,IAAI,QAAQ,KAAK;YACnE,mFAAmF;YACnF,gEAAgE,CACjE,CAAC;IACJ,CAAC;IAED,gDAAgD;IAChD,IAAI,QAAgB,CAAC;IACrB,IAAI,CAAC;QACH,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,CAAC;QAClC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,CAAC;YACnB,MAAM,IAAI,KAAK,CAAC,SAAS,QAAQ,0BAA0B,CAAC,CAAC;QAC/D,CAAC;QACD,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC;IACvB,CAAC;IAAC,OAAO,GAAY,EAAE,CAAC;QACtB,IAAK,GAA6B,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;YACrD,MAAM,IAAI,KAAK,CAAC,oBAAoB,QAAQ,kCAAkC,CAAC,CAAC;QAClF,CAAC;QACD,IAAK,GAA6B,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;YACrD,MAAM,IAAI,KAAK,CAAC,8BAA8B,QAAQ,IAAI,CAAC,CAAC;QAC9D,CAAC;QACD,MAAM,GAAG,CAAC;IACZ,CAAC;IAED,gEAAgE;IAChE,IAAI,QAAQ,GAAG,cAAc,EAAE,CAAC;QAC9B,MAAM,IAAI,KAAK,CACb,SAAS,QAAQ,CAAC,QAAQ,CAAC,QAAQ,CAAC,QAAQ,GAAG,IAAI,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO;YAC7E,0CAA0C,CAC3C,CAAC;IACJ,CAAC;IAED,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,gFAAgF;AAEhF;;;;;;;;;;;GAWG;AACH,MAAM,UAAU,qBAAqB,CAAC,IAAwB;IAC5D,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC;QAChC,OAAO,IAAI,GAAG,CAAC,oBAAoB,CAAC,CAAC;IACvC,CAAC;IAED,2EAA2E;IAC3E,MAAM,KAAK,GAAG,mBAAmB,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;IACpD,IAAI,CAAC,KAAK,EAAE,CAAC;QACX,mEAAmE;QACnE,OAAO,IAAI,GAAG,CAAC,oBAAoB,CAAC,CAAC;IACvC,CAAC;IAED,MAAM,GAAG,GAAG,KAAK,CAAC,CAAC,CAAE,CAAC,WAAW,EAAE,CAAC;IACpC,IAAI,CAAC,oBAAoB,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;QACnC,MAAM,IAAI,KAAK,CACb,iBAAiB,IAAI,yBAAyB,GAAG,4BAA4B;YAC7E,yBAAyB,CAAC,GAAG,oBAAoB,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI;YACjE,0DAA0D,CAC3D,CAAC;IACJ,CAAC;IAED,OAAO,IAAI,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;AACxB,CAAC;AAED;;;;;;;;;;;;;GAaG;AACH,MAAM,CAAC,KAAK,UAAU,eAAe,CACnC,UAAkB,EAClB,WAAwB;IAExB,IAAI,OAAyB,CAAC;IAC9B,IAAI,CAAC;QACH,OAAO,GAAG,MAAM,OAAO,CAAC,UAAU,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAqB,CAAC;IACnF,CAAC;IAAC,OAAO,GAAY,EAAE,CAAC;QACtB,IAAK,GAA6B,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;YACrD,MAAM,IAAI,KAAK,CAAC,sBAAsB,UAAU,kCAAkC,CAAC,CAAC;QACtF,CAAC;QACD,IAAK,GAA6B,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;YACtD,MAAM,IAAI,KAAK,CAAC,SAAS,UAAU,uBAAuB,CAAC,CAAC;QAC9D,CAAC;QACD,IAAK,GAA6B,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;YACrD,MAAM,IAAI,KAAK,CAAC,qCAAqC,UAAU,IAAI,CAAC,CAAC;QACvE,CAAC;QACD,MAAM,GAAG,CAAC;IACZ,CAAC;IAED,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;QAC5B,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE;YAAE,SAAS;QAC9B,MAAM,GAAG,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,WAAW,EAAE,CAAC;QAC9C,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,GAAG,CAAC;YAAE,SAAS;QACpC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC;IAC3C,CAAC;IAED,sEAAsE;IACtE,KAAK,CAAC,IAAI,EAAE,CAAC;IACb,OAAO,KAAK,CAAC;AACf,CAAC;AAED,kFAAkF;AAElF;;;;;;;;;;;GAWG;AACH,MAAM,UAAU,iBAAiB,CAAC,OAAe;IAC/C,OAAO,OAAO;SACX,OAAO,CAAC,OAAO,EAAE,IAAI,CAAC,CAAU,YAAY;SAC5C,OAAO,CAAC,KAAK,EAAE,IAAI,CAAC,CAAa,eAAe;SAChD,KAAK,CAAC,IAAI,CAAC;SACX,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC,CAAM,qCAAqC;SACtE,IAAI,CAAC,IAAI,CAAC;SACV,OAAO,CAAC,SAAS,EAAE,MAAM,CAAC,CAAO,+BAA+B;SAChE,IAAI,EAAE,CAAC;AACZ,CAAC;AAED,iFAAiF;AAEjF;;;;;;;;;;GAUG;AACH,MAAM,UAAU,YAAY,CAAC,OAAe;IAC1C,IAAI,OAAO,CAAC,MAAM,IAAI,gBAAgB,EAAE,CAAC;QACvC,OAAO,CAAC,OAAO,CAAC,CAAC;IACnB,CAAC;IAED,MAAM,UAAU,GAAG,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;IAC1C,MAAM,MAAM,GAAa,EAAE,CAAC;IAC5B,IAAI,OAAO,GAAG,EAAE,CAAC;IAEjB,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE,CAAC;QACnC,MAAM,SAAS,GAAG,OAAO,CAAC,CAAC,CAAC,GAAG,OAAO,OAAO,SAAS,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC;QAErE,IAAI,SAAS,CAAC,MAAM,GAAG,gBAAgB,IAAI,OAAO,EAAE,CAAC;YACnD,uBAAuB;YACvB,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YAErB,qEAAqE;YACrE,MAAM,YAAY,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,CAAC,MAAM,GAAG,mBAAmB,CAAC,CAAC;YACvE,MAAM,OAAO,GAAG,OAAO,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC;YAC5C,OAAO,GAAG,GAAG,OAAO,OAAO,SAAS,EAAE,CAAC;QACzC,CAAC;aAAM,CAAC;YACN,OAAO,GAAG,SAAS,CAAC;QACtB,CAAC;IACH,CAAC;IAED,IAAI,OAAO,EAAE,CAAC;QACZ,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IACvB,CAAC;IAED,OAAO,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;AAChD,CAAC;AAED,iFAAiF;AAEjF,kDAAkD;AAClD,MAAM,UAAU,MAAM,CAAC,OAAe;IACpC,OAAO,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;AACpE,CAAC;AA8BD;;;;;;;;;;;;;GAaG;AACH,KAAK,UAAU,YAAY,CACzB,UAAkB,EAClB,UAAkB,EAClB,KAAa,EACb,OAAsB;IAEtB,MAAM,GAAG,GAAG,GAAG,UAAU,WAAW,kBAAkB,CAAC,UAAU,CAAC,EAAE,CAAC;IAErE,IAAI,GAAa,CAAC;IAClB,IAAI,CAAC;QACH,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE;YACrB,MAAM,EAAE,MAAM;YACd,OAAO,EAAE;gBACP,cAAc,EAAE,kBAAkB;gBAClC,eAAe,EAAE,UAAU,KAAK,EAAE;aACnC;YACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC;SAC9B,CAAC,CAAC;IACL,CAAC;IAAC,OAAO,GAAY,EAAE,CAAC;QACtB,kEAAkE;QAClE,OAAO;YACL,IAAI,EAAE,WAAW;YACjB,KAAK,EACH,0BAA0B,GAAG,KAAM,GAAa,CAAC,OAAO,IAAI;gBAC5D,8CAA8C;SACjD,CAAC;IACJ,CAAC;IAED,IAAI,QAAQ,GAAG,EAAE,CAAC;IAClB,IAAI,CAAC;QACH,QAAQ,GAAG,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC;IAC9B,CAAC;IAAC,MAAM,CAAC,CAAC,YAAY,CAAC,CAAC;IAExB,4EAA4E;IAC5E,IAAI,GAAG,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;QACvB,OAAO;YACL,IAAI,EAAE,WAAW;YACjB,KAAK,EAAE,6FAA6F;SACrG,CAAC;IACJ,CAAC;IAED,6CAA6C;IAC7C,IAAI,GAAG,CAAC,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC;QAC1C,IAAI,IAAI,GAAG,EAAE,CAAC;QACd,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,CAA4B,CAAC;YAC/D,IAAI,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,IAAI,MAAM,CAAC,SAAS,CAAC,IAAI,EAAE,CAAW,CAAC;QAClF,CAAC;QAAC,MAAM,CAAC,CAAC,cAAc,CAAC,CAAC;QAE1B,IAAI,GAAG,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;YACvB,OAAO;gBACL,IAAI,EAAE,WAAW;gBACjB,KAAK,EAAE,oGAAoG;aAC5G,CAAC;QACJ,CAAC;QACD,IAAI,GAAG,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;YACvB,OAAO;gBACL,IAAI,EAAE,WAAW;gBACjB,KAAK,EAAE,4BAA4B,UAAU,YAAY,IAAI,IAAI,uCAAuC,EAAE;aAC3G,CAAC;QACJ,CAAC;QACD,OAAO;YACL,IAAI,EAAE,WAAW;YACjB,KAAK,EAAE,yBAAyB,GAAG,CAAC,MAAM,GAAG,IAAI,CAAC,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE;SAClG,CAAC;IACJ,CAAC;IAED,oCAAoC;IACpC,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC;QACZ,OAAO;YACL,IAAI,EAAE,WAAW;YACjB,KAAK,EAAE,yBAAyB,GAAG,CAAC,MAAM,KAAK,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE;SACxE,CAAC;IACJ,CAAC;IAED,eAAe;IACf,IAAI,CAAC;QACH,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,QAAQ,EAAE,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAmB,EAAE,CAAC;IAC/E,CAAC;IAAC,MAAM,CAAC;QACP,0DAA0D;QAC1D,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,QAAQ,EAAE,EAAE,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,EAAE,CAAC;IACrE,CAAC;AACH,CAAC;AAED;;;;;;;;GAQG;AACH,KAAK,UAAU,SAAS,CACtB,UAAkB,EAClB,UAAkB,EAClB,KAAa,EACb,OAAsB;IAEtB,MAAM,OAAO,GAAG,MAAM,YAAY,CAAC,UAAU,EAAE,UAAU,EAAE,KAAK,EAAE,OAAO,CAAC,CAAC;IAC3E,IAAI,OAAO,CAAC,IAAI,KAAK,SAAS;QAAE,OAAO,OAAO,CAAC,QAAQ,CAAC;IACxD,MAAM,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;AACjC,CAAC;AAED,iFAAiF;AAEjF;;;;;;;;;;;;;;;;GAgBG;AACH,MAAM,CAAC,KAAK,UAAU,kBAAkB,CAAC,IAAuB;IAC9D,MAAM,EAAE,QAAQ,EAAE,UAAU,EAAE,UAAU,EAAE,KAAK,EAAE,cAAc,EAAE,WAAW,EAAE,cAAc,EAAE,GAAG,IAAI,CAAC;IAEtG,0DAA0D;IAC1D,MAAM,YAAY,GAAG,MAAM,sBAAsB,CAAC,QAAQ,CAAC,CAAC;IAC5D,MAAM,QAAQ,GAAG,QAAQ,CAAC,YAAY,CAAC,CAAC;IAExC,0EAA0E;IAC1E,MAAM,GAAG,GAAG,OAAO,CAAC,YAAY,CAAC,CAAC,WAAW,EAAE,CAAC;IAChD,MAAM,UAAU,GAAG,gBAAgB,CAAC,GAAG,CAAC,CAAC;IAEzC,aAAa;IACb,MAAM,UAAU,GAAG,MAAM,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC,CAAC;IAExD,4DAA4D;IAC5D,MAAM,UAAU,GAAG,iBAAiB,CAAC,UAAU,CAAC,CAAC;IAEjD,IAAI,CAAC,UAAU,EAAE,CAAC;QAChB,MAAM,IAAI,KAAK,CAAC,SAAS,QAAQ,oDAAoD,CAAC,CAAC;IACzF,CAAC;IAED,gFAAgF;IAChF,MAAM,WAAW,GAAG,MAAM,CAAC,UAAU,CAAC,CAAC;IAEvC,SAAS;IACT,MAAM,MAAM,GAAG,YAAY,CAAC,UAAU,CAAC,CAAC;IAExC,8EAA8E;IAC9E,MAAM,oBAAoB,GAAG,WAAW,IAAI,EAAE,CAAC;IAC/C,MAAM,SAAS,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IAE3C,IAAI,cAAc,GAAG,CAAC,CAAC;IACvB,IAAI,aAAa,GAAG,CAAC,CAAC;IACtB,IAAI,YAAY,GAAG,CAAC,CAAC;IAErB,mCAAmC;IACnC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACvC,MAAM,SAAS,GAAG,MAAM,CAAC,CAAC,CAAE,CAAC;QAC7B,MAAM,SAAS,GAAG,MAAM,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;QAExE,mEAAmE;QACnE,MAAM,aAAa,GAAG,MAAM,CAAC,MAAM,KAAK,CAAC;YACvC,CAAC,CAAC,QAAQ;YACV,CAAC,CAAC,GAAG,QAAQ,UAAU,CAAC,GAAG,CAAC,OAAO,MAAM,CAAC,MAAM,EAAE,CAAC;QAErD,MAAM,OAAO,GAAkB;YAC7B,WAAW,EAAE,UAAU;YACvB,QAAQ,EAAE,aAAa;YACvB,OAAO,EAAE,SAAS;YAClB,MAAM,EAAE,SAAS;YACjB,SAAS;YACT,YAAY,EAAE,oBAAoB;YAClC,GAAG,CAAC,cAAc,CAAC,CAAC,CAAC,EAAE,eAAe,EAAE,cAAc,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SAC/D,CAAC;QAEF,IAAI,cAAc,EAAE,CAAC;YACnB,qFAAqF;YACrF,MAAM,OAAO,GAAG,MAAM,YAAY,CAAC,UAAU,EAAE,UAAU,EAAE,KAAK,EAAE,OAAO,CAAC,CAAC;YAE3E,IAAI,OAAO,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;gBAC/B,IAAI,OAAO,CAAC,QAAQ,CAAC,OAAO,EAAE,CAAC;oBAC7B,aAAa,EAAE,CAAC;gBAClB,CAAC;qBAAM,CAAC;oBACN,cAAc,EAAE,CAAC;gBACnB,CAAC;YACH,CAAC;iBAAM,IAAI,OAAO,CAAC,IAAI,KAAK,WAAW,EAAE,CAAC;gBACxC,4FAA4F;gBAC5F,MAAM,kBAAkB,CAAC,UAAU,EAAE,UAAU,EAAE,OAAO,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC;gBACzE,YAAY,EAAE,CAAC;YACjB,CAAC;iBAAM,CAAC;gBACN,gFAAgF;gBAChF,MAAM,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;YACjC,CAAC;QACH,CAAC;aAAM,CAAC;YACN,4CAA4C;YAC5C,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,UAAU,EAAE,UAAU,EAAE,KAAK,EAAE,OAAO,CAAC,CAAC;YACvE,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;gBACnB,aAAa,EAAE,CAAC;YAClB,CAAC;iBAAM,CAAC;gBACN,cAAc,EAAE,CAAC;YACnB,CAAC;QACH,CAAC;IACH,CAAC;IAED,kBAAkB;IAClB,OAAO;QACL,WAAW,EAAE,MAAM,CAAC,MAAM;QAC1B,eAAe,EAAE,cAAc;QAC/B,cAAc,EAAE,aAAa;QAC7B,aAAa,EAAE,YAAY;QAC3B,IAAI,EAAE,YAAY;QAClB,YAAY,EAAE,WAAW;KAC1B,CAAC;AACJ,CAAC;AAED,iFAAiF;AAEjF;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,IAAyB;IAC1D,MAAM,EAAE,UAAU,EAAE,IAAI,EAAE,UAAU,EAAE,UAAU,EAAE,KAAK,EAAE,cAAc,EAAE,WAAW,EAAE,GAAG,IAAI,CAAC;IAE9F,oCAAoC;IACpC,MAAM,cAAc,GAAG,OAAO,CAAC,UAAU,CAAC,IAAI,EAAE,CAAC,CAAC;IAElD,+CAA+C;IAC/C,IAAI,CAAC;QACH,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,CAAC;QACxC,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,EAAE,CAAC;YACxB,MAAM,IAAI,KAAK,CAAC,SAAS,cAAc,uBAAuB,CAAC,CAAC;QAClE,CAAC;IACH,CAAC;IAAC,OAAO,GAAY,EAAE,CAAC;QACtB,IAAK,GAA6B,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;YACrD,MAAM,IAAI,KAAK,CAAC,sBAAsB,cAAc,kCAAkC,CAAC,CAAC;QAC1F,CAAC;QACD,IAAK,GAA6B,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;YACrD,MAAM,IAAI,KAAK,CAAC,qCAAqC,cAAc,IAAI,CAAC,CAAC;QAC3E,CAAC;QACD,MAAM,GAAG,CAAC;IACZ,CAAC;IAED,4CAA4C;IAC5C,MAAM,WAAW,GAAG,qBAAqB,CAAC,IAAI,CAAC,CAAC;IAEhD,kCAAkC;IAClC,MAAM,KAAK,GAAG,MAAM,eAAe,CAAC,cAAc,EAAE,WAAW,CAAC,CAAC;IAEjE,MAAM,OAAO,GAAuB,EAAE,CAAC;IACvC,IAAI,aAAa,GAAG,CAAC,CAAC;IACtB,IAAI,YAAY,GAAG,CAAC,CAAC;IACrB,IAAI,WAAW,GAAG,CAAC,CAAC;IAEpB,6DAA6D;IAC7D,KAAK,MAAM,QAAQ,IAAI,KAAK,EAAE,CAAC;QAC7B,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,MAAM,kBAAkB,CAAC;gBACvC,QAAQ;gBACR,UAAU;gBACV,UAAU;gBACV,KAAK;gBACL,cAAc;gBACd,WAAW;aACZ,CAAC,CAAC;YAEH,0EAA0E;YAC1E,MAAM,YAAY,GAAG,OAAO,CAAC,eAAe,KAAK,CAAC,IAAI,OAAO,CAAC,cAAc,GAAG,CAAC,CAAC;YAEjF,IAAI,YAAY,EAAE,CAAC;gBACjB,YAAY,EAAE,CAAC;gBACf,OAAO,CAAC,IAAI,CAAC;oBACX,IAAI,EAAE,QAAQ;oBACd,MAAM,EAAE,SAAS;oBACjB,WAAW,EAAE,OAAO,CAAC,WAAW;oBAChC,cAAc,EAAE,OAAO,CAAC,cAAc;iBACvC,CAAC,CAAC;YACL,CAAC;iBAAM,CAAC;gBACN,aAAa,EAAE,CAAC;gBAChB,OAAO,CAAC,IAAI,CAAC;oBACX,IAAI,EAAE,QAAQ;oBACd,MAAM,EAAE,UAAU;oBAClB,WAAW,EAAE,OAAO,CAAC,WAAW;oBAChC,eAAe,EAAE,OAAO,CAAC,eAAe;oBACxC,cAAc,EAAE,OAAO,CAAC,cAAc;iBACvC,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QAAC,OAAO,GAAY,EAAE,CAAC;YACtB,oEAAoE;YACpE,WAAW,EAAE,CAAC;YACd,OAAO,CAAC,IAAI,CAAC;gBACX,IAAI,EAAE,QAAQ;gBACd,MAAM,EAAE,QAAQ;gBAChB,KAAK,EAAE,CAAC,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC;aAC1D,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,OAAO;QACL,MAAM,EAAE,cAAc;QACtB,WAAW,EAAE,KAAK,CAAC,MAAM;QACzB,cAAc,EAAE,aAAa;QAC7B,aAAa,EAAE,YAAY;QAC3B,YAAY,EAAE,WAAW;QACzB,OAAO;KACR,CAAC;AACJ,CAAC"}
|
package/dist/proxy.d.ts
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Backend proxy module for the Sound Connect bridge (STORY-008, ADR-002, ADR-003).
|
|
3
|
+
*
|
|
4
|
+
* Implements two operations against the backend `/mcp/:slug` endpoint:
|
|
5
|
+
* 1. fetchBackendTools — issues a `tools/list` JSON-RPC call and returns the tool list.
|
|
6
|
+
* 2. callBackendTool — issues a `tools/call` JSON-RPC call and returns the result.
|
|
7
|
+
*
|
|
8
|
+
* Both operations authenticate via `Authorization: Bearer <token>` (ADR-003).
|
|
9
|
+
* The backend uses StreamableHTTPServerTransport in stateless mode — every call is an
|
|
10
|
+
* independent HTTP POST; no session handshake is required.
|
|
11
|
+
*
|
|
12
|
+
* ADR-011 error handling:
|
|
13
|
+
* - 401 → BackendAuthError (token invalid / expired — user must re-login)
|
|
14
|
+
* - 403 → BackendForbiddenError (peer not a member of this client)
|
|
15
|
+
* - 429 → BackendRateLimitError (back off and retry)
|
|
16
|
+
* - 5xx → BackendUnavailableError (backend unreachable / internal error)
|
|
17
|
+
* - JSON parse failure → BackendProtocolError (unexpected response shape)
|
|
18
|
+
*
|
|
19
|
+
* These are surfaced as MCP tool errors in tools.ts — never as silent failures.
|
|
20
|
+
*/
|
|
21
|
+
/** A single tool entry as returned by the backend `tools/list` response. */
|
|
22
|
+
export interface BackendTool {
|
|
23
|
+
name: string;
|
|
24
|
+
description?: string;
|
|
25
|
+
inputSchema: {
|
|
26
|
+
type: 'object';
|
|
27
|
+
properties?: Record<string, object>;
|
|
28
|
+
required?: string[];
|
|
29
|
+
};
|
|
30
|
+
}
|
|
31
|
+
/** Result of a `tools/call` forwarded to the backend. */
|
|
32
|
+
export interface BackendToolResult {
|
|
33
|
+
content: Array<{
|
|
34
|
+
type: string;
|
|
35
|
+
text?: string;
|
|
36
|
+
[key: string]: unknown;
|
|
37
|
+
}>;
|
|
38
|
+
isError?: boolean;
|
|
39
|
+
}
|
|
40
|
+
/** The bearer token was rejected (401). Peer must re-run `login`. */
|
|
41
|
+
export declare class BackendAuthError extends Error {
|
|
42
|
+
constructor(msg?: string);
|
|
43
|
+
}
|
|
44
|
+
/** Peer is not a member of the requested client (403). */
|
|
45
|
+
export declare class BackendForbiddenError extends Error {
|
|
46
|
+
constructor(slug: string, hint?: string);
|
|
47
|
+
}
|
|
48
|
+
/** Backend is rate-limiting this peer (429). */
|
|
49
|
+
export declare class BackendRateLimitError extends Error {
|
|
50
|
+
constructor(retryAfterMs?: number);
|
|
51
|
+
}
|
|
52
|
+
/** Backend returned a 5xx or was unreachable. */
|
|
53
|
+
export declare class BackendUnavailableError extends Error {
|
|
54
|
+
constructor(status: number, msg?: string);
|
|
55
|
+
}
|
|
56
|
+
/** The backend response did not conform to the expected JSON-RPC shape. */
|
|
57
|
+
export declare class BackendProtocolError extends Error {
|
|
58
|
+
constructor(detail: string);
|
|
59
|
+
}
|
|
60
|
+
/**
|
|
61
|
+
* Fetch the backend's tool list for the bound client.
|
|
62
|
+
*
|
|
63
|
+
* Sends `tools/list` JSON-RPC to `/mcp/:slug` with the peer's bearer token.
|
|
64
|
+
* Returns the tool array from the response; throws a typed BackendXxxError on failure.
|
|
65
|
+
*
|
|
66
|
+
* STORY-008 AC1: bridge fetches the backend tool list for the bound client.
|
|
67
|
+
*/
|
|
68
|
+
export declare function fetchBackendTools(backendUrl: string, slug: string, token: string): Promise<BackendTool[]>;
|
|
69
|
+
/**
|
|
70
|
+
* Forward a tool call to the backend `/mcp/:slug` endpoint.
|
|
71
|
+
*
|
|
72
|
+
* Sends `tools/call` JSON-RPC with the provided arguments, authenticated via bearer token.
|
|
73
|
+
* Parses the result and returns it as a BackendToolResult.
|
|
74
|
+
*
|
|
75
|
+
* STORY-008 AC2: tool calls forwarded with Authorization: Bearer <token>.
|
|
76
|
+
* STORY-008 AC4: backend errors surface as MCP tool errors (isError: true), not exceptions.
|
|
77
|
+
*/
|
|
78
|
+
export declare function callBackendTool(backendUrl: string, slug: string, token: string, toolName: string, toolArgs: Record<string, unknown>): Promise<BackendToolResult>;
|
|
79
|
+
//# sourceMappingURL=proxy.d.ts.map
|