@indigoai-us/hq-cloud 5.19.1 → 5.20.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.github/workflows/ci.yml +8 -4
- package/.github/workflows/publish.yml +9 -3
- package/dist/bin/sync-runner.d.ts +9 -0
- package/dist/bin/sync-runner.d.ts.map +1 -1
- package/dist/bin/sync-runner.js +58 -0
- package/dist/bin/sync-runner.js.map +1 -1
- package/dist/entity-resolver.d.ts +48 -0
- package/dist/entity-resolver.d.ts.map +1 -0
- package/dist/entity-resolver.js +122 -0
- package/dist/entity-resolver.js.map +1 -0
- package/dist/entity-resolver.test.d.ts +10 -0
- package/dist/entity-resolver.test.d.ts.map +1 -0
- package/dist/entity-resolver.test.js +236 -0
- package/dist/entity-resolver.test.js.map +1 -0
- package/dist/index.d.ts +18 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +21 -0
- package/dist/index.js.map +1 -1
- package/dist/schemas/signal-types.d.ts +16 -0
- package/dist/schemas/signal-types.d.ts.map +1 -0
- package/dist/schemas/signal-types.js +30 -0
- package/dist/schemas/signal-types.js.map +1 -0
- package/dist/schemas/signal-types.test.d.ts +2 -0
- package/dist/schemas/signal-types.test.d.ts.map +1 -0
- package/dist/schemas/signal-types.test.js +65 -0
- package/dist/schemas/signal-types.test.js.map +1 -0
- package/dist/schemas/source-channels.d.ts +15 -0
- package/dist/schemas/source-channels.d.ts.map +1 -0
- package/dist/schemas/source-channels.js +28 -0
- package/dist/schemas/source-channels.js.map +1 -0
- package/dist/schemas/source-channels.test.d.ts +2 -0
- package/dist/schemas/source-channels.test.d.ts.map +1 -0
- package/dist/schemas/source-channels.test.js +65 -0
- package/dist/schemas/source-channels.test.js.map +1 -0
- package/dist/signals/get.d.ts +13 -0
- package/dist/signals/get.d.ts.map +1 -0
- package/dist/signals/get.js +74 -0
- package/dist/signals/get.js.map +1 -0
- package/dist/signals/get.test.d.ts +5 -0
- package/dist/signals/get.test.d.ts.map +1 -0
- package/dist/signals/get.test.js +170 -0
- package/dist/signals/get.test.js.map +1 -0
- package/dist/signals/internals.d.ts +16 -0
- package/dist/signals/internals.d.ts.map +1 -0
- package/dist/signals/internals.js +39 -0
- package/dist/signals/internals.js.map +1 -0
- package/dist/signals/list.d.ts +10 -0
- package/dist/signals/list.d.ts.map +1 -0
- package/dist/signals/list.js +76 -0
- package/dist/signals/list.js.map +1 -0
- package/dist/signals/list.test.d.ts +9 -0
- package/dist/signals/list.test.d.ts.map +1 -0
- package/dist/signals/list.test.js +227 -0
- package/dist/signals/list.test.js.map +1 -0
- package/dist/signals/parse.d.ts +8 -0
- package/dist/signals/parse.d.ts.map +1 -0
- package/dist/signals/parse.js +8 -0
- package/dist/signals/parse.js.map +1 -0
- package/dist/signals/types.d.ts +69 -0
- package/dist/signals/types.d.ts.map +1 -0
- package/dist/signals/types.js +10 -0
- package/dist/signals/types.js.map +1 -0
- package/dist/sources/get.d.ts +11 -0
- package/dist/sources/get.d.ts.map +1 -0
- package/dist/sources/get.js +67 -0
- package/dist/sources/get.js.map +1 -0
- package/dist/sources/get.test.d.ts +5 -0
- package/dist/sources/get.test.d.ts.map +1 -0
- package/dist/sources/get.test.js +132 -0
- package/dist/sources/get.test.js.map +1 -0
- package/dist/sources/internals.d.ts +16 -0
- package/dist/sources/internals.d.ts.map +1 -0
- package/dist/sources/internals.js +39 -0
- package/dist/sources/internals.js.map +1 -0
- package/dist/sources/list.d.ts +10 -0
- package/dist/sources/list.d.ts.map +1 -0
- package/dist/sources/list.js +76 -0
- package/dist/sources/list.js.map +1 -0
- package/dist/sources/list.test.d.ts +8 -0
- package/dist/sources/list.test.d.ts.map +1 -0
- package/dist/sources/list.test.js +198 -0
- package/dist/sources/list.test.js.map +1 -0
- package/dist/sources/parse.d.ts +18 -0
- package/dist/sources/parse.d.ts.map +1 -0
- package/dist/sources/parse.js +35 -0
- package/dist/sources/parse.js.map +1 -0
- package/dist/sources/types.d.ts +62 -0
- package/dist/sources/types.d.ts.map +1 -0
- package/dist/sources/types.js +8 -0
- package/dist/sources/types.js.map +1 -0
- package/dist/telemetry.d.ts +87 -0
- package/dist/telemetry.d.ts.map +1 -0
- package/dist/telemetry.js +349 -0
- package/dist/telemetry.js.map +1 -0
- package/dist/telemetry.test.d.ts +11 -0
- package/dist/telemetry.test.d.ts.map +1 -0
- package/dist/telemetry.test.js +309 -0
- package/dist/telemetry.test.js.map +1 -0
- package/dist/vault-client.d.ts +59 -0
- package/dist/vault-client.d.ts.map +1 -1
- package/dist/vault-client.js +37 -0
- package/dist/vault-client.js.map +1 -1
- package/package.json +5 -3
- package/src/bin/sync-runner.ts +73 -0
- package/src/entity-resolver.test.ts +307 -0
- package/src/entity-resolver.ts +173 -0
- package/src/index.ts +79 -0
- package/src/schemas/signal-types.test.ts +82 -0
- package/src/schemas/signal-types.ts +38 -0
- package/src/schemas/source-channels.test.ts +82 -0
- package/src/schemas/source-channels.ts +36 -0
- package/src/signals/get.test.ts +204 -0
- package/src/signals/get.ts +79 -0
- package/src/signals/internals.ts +46 -0
- package/src/signals/list.test.ts +283 -0
- package/src/signals/list.ts +92 -0
- package/src/signals/parse.ts +8 -0
- package/src/signals/types.ts +74 -0
- package/src/sources/get.test.ts +166 -0
- package/src/sources/get.ts +75 -0
- package/src/sources/internals.ts +46 -0
- package/src/sources/list.test.ts +247 -0
- package/src/sources/list.ts +95 -0
- package/src/sources/parse.ts +43 -0
- package/src/sources/types.ts +67 -0
- package/src/telemetry.test.ts +394 -0
- package/src/telemetry.ts +436 -0
- package/src/vault-client.ts +86 -0
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Internal helpers shared between sources/list.ts and sources/get.ts.
|
|
3
|
+
*
|
|
4
|
+
* The S3 client factory hook exists so tests can inject an in-memory stub
|
|
5
|
+
* without standing up vi.mock for @aws-sdk/client-s3. Production code never
|
|
6
|
+
* calls the setter.
|
|
7
|
+
*/
|
|
8
|
+
import { S3Client } from "@aws-sdk/client-s3";
|
|
9
|
+
function defaultFactory(ctx) {
|
|
10
|
+
return new S3Client({
|
|
11
|
+
region: ctx.region,
|
|
12
|
+
credentials: {
|
|
13
|
+
accessKeyId: ctx.credentials.accessKeyId,
|
|
14
|
+
secretAccessKey: ctx.credentials.secretAccessKey,
|
|
15
|
+
sessionToken: ctx.credentials.sessionToken,
|
|
16
|
+
},
|
|
17
|
+
});
|
|
18
|
+
}
|
|
19
|
+
let factory = defaultFactory;
|
|
20
|
+
export function getS3Client(ctx) {
|
|
21
|
+
return factory(ctx);
|
|
22
|
+
}
|
|
23
|
+
/** Test hook: replace the S3 client factory. */
|
|
24
|
+
export function _setSourcesS3Factory(f) {
|
|
25
|
+
factory = f;
|
|
26
|
+
}
|
|
27
|
+
/** Test hook: restore the default S3 client factory. */
|
|
28
|
+
export function _resetSourcesS3Factory() {
|
|
29
|
+
factory = defaultFactory;
|
|
30
|
+
}
|
|
31
|
+
export async function streamToString(body) {
|
|
32
|
+
const stream = body;
|
|
33
|
+
const chunks = [];
|
|
34
|
+
for await (const chunk of stream) {
|
|
35
|
+
chunks.push(Buffer.from(chunk));
|
|
36
|
+
}
|
|
37
|
+
return Buffer.concat(chunks).toString("utf-8");
|
|
38
|
+
}
|
|
39
|
+
//# sourceMappingURL=internals.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"internals.js","sourceRoot":"","sources":["../../src/sources/internals.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AAG9C,SAAS,cAAc,CAAC,GAAkB;IACxC,OAAO,IAAI,QAAQ,CAAC;QAClB,MAAM,EAAE,GAAG,CAAC,MAAM;QAClB,WAAW,EAAE;YACX,WAAW,EAAE,GAAG,CAAC,WAAW,CAAC,WAAW;YACxC,eAAe,EAAE,GAAG,CAAC,WAAW,CAAC,eAAe;YAChD,YAAY,EAAE,GAAG,CAAC,WAAW,CAAC,YAAY;SAC3C;KACF,CAAC,CAAC;AACL,CAAC;AAED,IAAI,OAAO,GAAqC,cAAc,CAAC;AAE/D,MAAM,UAAU,WAAW,CAAC,GAAkB;IAC5C,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC;AACtB,CAAC;AAED,gDAAgD;AAChD,MAAM,UAAU,oBAAoB,CAAC,CAAmC;IACtE,OAAO,GAAG,CAAC,CAAC;AACd,CAAC;AAED,wDAAwD;AACxD,MAAM,UAAU,sBAAsB;IACpC,OAAO,GAAG,cAAc,CAAC;AAC3B,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,cAAc,CAAC,IAAa;IAChD,MAAM,MAAM,GAAG,IAAiC,CAAC;IACjD,MAAM,MAAM,GAAa,EAAE,CAAC;IAC5B,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QACjC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;IAClC,CAAC;IACD,OAAO,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;AACjD,CAAC"}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* listSources — page through `sources/{channel}/` keys in an entity bucket
|
|
3
|
+
* and return typed summaries.
|
|
4
|
+
*
|
|
5
|
+
* The `.raw.json` siblings written by the sources-pipeline are skipped from
|
|
6
|
+
* the summary (callers fetch them via getSource({ includeRaw: true })).
|
|
7
|
+
*/
|
|
8
|
+
import type { ListSourcesOptions, ListSourcesResult } from "./types.js";
|
|
9
|
+
export declare function listSources(opts: ListSourcesOptions): Promise<ListSourcesResult>;
|
|
10
|
+
//# sourceMappingURL=list.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"list.d.ts","sourceRoot":"","sources":["../../src/sources/list.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AASH,OAAO,KAAK,EACV,kBAAkB,EAClB,iBAAiB,EAElB,MAAM,YAAY,CAAC;AAsBpB,wBAAsB,WAAW,CAAC,IAAI,EAAE,kBAAkB,GAAG,OAAO,CAAC,iBAAiB,CAAC,CAqDtF"}
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* listSources — page through `sources/{channel}/` keys in an entity bucket
|
|
3
|
+
* and return typed summaries.
|
|
4
|
+
*
|
|
5
|
+
* The `.raw.json` siblings written by the sources-pipeline are skipped from
|
|
6
|
+
* the summary (callers fetch them via getSource({ includeRaw: true })).
|
|
7
|
+
*/
|
|
8
|
+
import { GetObjectCommand, ListObjectsV2Command, } from "@aws-sdk/client-s3";
|
|
9
|
+
import { assertSourceChannel } from "../schemas/source-channels.js";
|
|
10
|
+
import { getS3Client, streamToString } from "./internals.js";
|
|
11
|
+
import { parseFrontmatter } from "./parse.js";
|
|
12
|
+
/**
|
|
13
|
+
* Strip the `sources/{channel}/` prefix and `.md` / `.raw.json` suffix from a key.
|
|
14
|
+
* Returns `null` if the key shape isn't recognised; the `isRawSibling` flag tells
|
|
15
|
+
* callers to skip pipeline-internal `.raw.json` companions from summaries.
|
|
16
|
+
*/
|
|
17
|
+
function deriveSourceId(key, prefix) {
|
|
18
|
+
if (!key.startsWith(prefix))
|
|
19
|
+
return null;
|
|
20
|
+
const tail = key.slice(prefix.length);
|
|
21
|
+
if (tail.endsWith(".raw.json")) {
|
|
22
|
+
return { sourceId: tail.slice(0, -".raw.json".length), isRawSibling: true };
|
|
23
|
+
}
|
|
24
|
+
if (tail.endsWith(".md")) {
|
|
25
|
+
return { sourceId: tail.slice(0, -".md".length), isRawSibling: false };
|
|
26
|
+
}
|
|
27
|
+
return null;
|
|
28
|
+
}
|
|
29
|
+
export async function listSources(opts) {
|
|
30
|
+
// Validate channel BEFORE any S3 call (acceptance criterion).
|
|
31
|
+
assertSourceChannel(opts.channel);
|
|
32
|
+
const client = getS3Client(opts.entity);
|
|
33
|
+
const prefix = `sources/${opts.channel}/`;
|
|
34
|
+
const response = await client.send(new ListObjectsV2Command({
|
|
35
|
+
Bucket: opts.entity.bucketName,
|
|
36
|
+
Prefix: prefix,
|
|
37
|
+
MaxKeys: opts.limit,
|
|
38
|
+
ContinuationToken: opts.continuationToken,
|
|
39
|
+
}));
|
|
40
|
+
const entries = [];
|
|
41
|
+
for (const obj of response.Contents ?? []) {
|
|
42
|
+
if (!obj.Key)
|
|
43
|
+
continue;
|
|
44
|
+
const derived = deriveSourceId(obj.Key, prefix);
|
|
45
|
+
if (!derived || derived.isRawSibling)
|
|
46
|
+
continue;
|
|
47
|
+
const summary = {
|
|
48
|
+
sourceId: derived.sourceId,
|
|
49
|
+
channel: opts.channel,
|
|
50
|
+
key: obj.Key,
|
|
51
|
+
lastModified: obj.LastModified ?? new Date(0),
|
|
52
|
+
size: obj.Size ?? 0,
|
|
53
|
+
};
|
|
54
|
+
if (opts.includeFrontmatter) {
|
|
55
|
+
try {
|
|
56
|
+
const get = await client.send(new GetObjectCommand({
|
|
57
|
+
Bucket: opts.entity.bucketName,
|
|
58
|
+
Key: obj.Key,
|
|
59
|
+
}));
|
|
60
|
+
const text = await streamToString(get.Body);
|
|
61
|
+
const fm = parseFrontmatter(text);
|
|
62
|
+
if (fm)
|
|
63
|
+
summary.frontmatter = fm;
|
|
64
|
+
}
|
|
65
|
+
catch {
|
|
66
|
+
// Frontmatter fetch is best-effort; leave undefined on failure.
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
entries.push(summary);
|
|
70
|
+
}
|
|
71
|
+
return {
|
|
72
|
+
entries,
|
|
73
|
+
nextToken: response.IsTruncated ? response.NextContinuationToken : undefined,
|
|
74
|
+
};
|
|
75
|
+
}
|
|
76
|
+
//# sourceMappingURL=list.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"list.js","sourceRoot":"","sources":["../../src/sources/list.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,EACL,gBAAgB,EAChB,oBAAoB,GACrB,MAAM,oBAAoB,CAAC;AAC5B,OAAO,EAAE,mBAAmB,EAAE,MAAM,+BAA+B,CAAC;AACpE,OAAO,EAAE,WAAW,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAC;AAC7D,OAAO,EAAE,gBAAgB,EAAE,MAAM,YAAY,CAAC;AAO9C;;;;GAIG;AACH,SAAS,cAAc,CACrB,GAAW,EACX,MAAc;IAEd,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,MAAM,CAAC;QAAE,OAAO,IAAI,CAAC;IACzC,MAAM,IAAI,GAAG,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;IACtC,IAAI,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC,EAAE,CAAC;QAC/B,OAAO,EAAE,QAAQ,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,WAAW,CAAC,MAAM,CAAC,EAAE,YAAY,EAAE,IAAI,EAAE,CAAC;IAC9E,CAAC;IACD,IAAI,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;QACzB,OAAO,EAAE,QAAQ,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,YAAY,EAAE,KAAK,EAAE,CAAC;IACzE,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,IAAwB;IACxD,8DAA8D;IAC9D,mBAAmB,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IAElC,MAAM,MAAM,GAAG,WAAW,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IACxC,MAAM,MAAM,GAAG,WAAW,IAAI,CAAC,OAAO,GAAG,CAAC;IAE1C,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,IAAI,CAChC,IAAI,oBAAoB,CAAC;QACvB,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,UAAU;QAC9B,MAAM,EAAE,MAAM;QACd,OAAO,EAAE,IAAI,CAAC,KAAK;QACnB,iBAAiB,EAAE,IAAI,CAAC,iBAAiB;KAC1C,CAAC,CACH,CAAC;IAEF,MAAM,OAAO,GAAoB,EAAE,CAAC;IACpC,KAAK,MAAM,GAAG,IAAI,QAAQ,CAAC,QAAQ,IAAI,EAAE,EAAE,CAAC;QAC1C,IAAI,CAAC,GAAG,CAAC,GAAG;YAAE,SAAS;QACvB,MAAM,OAAO,GAAG,cAAc,CAAC,GAAG,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;QAChD,IAAI,CAAC,OAAO,IAAI,OAAO,CAAC,YAAY;YAAE,SAAS;QAE/C,MAAM,OAAO,GAAkB;YAC7B,QAAQ,EAAE,OAAO,CAAC,QAAQ;YAC1B,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,GAAG,EAAE,GAAG,CAAC,GAAG;YACZ,YAAY,EAAE,GAAG,CAAC,YAAY,IAAI,IAAI,IAAI,CAAC,CAAC,CAAC;YAC7C,IAAI,EAAE,GAAG,CAAC,IAAI,IAAI,CAAC;SACpB,CAAC;QAEF,IAAI,IAAI,CAAC,kBAAkB,EAAE,CAAC;YAC5B,IAAI,CAAC;gBACH,MAAM,GAAG,GAAG,MAAM,MAAM,CAAC,IAAI,CAC3B,IAAI,gBAAgB,CAAC;oBACnB,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,UAAU;oBAC9B,GAAG,EAAE,GAAG,CAAC,GAAG;iBACb,CAAC,CACH,CAAC;gBACF,MAAM,IAAI,GAAG,MAAM,cAAc,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;gBAC5C,MAAM,EAAE,GAAG,gBAAgB,CAAC,IAAI,CAAC,CAAC;gBAClC,IAAI,EAAE;oBAAE,OAAO,CAAC,WAAW,GAAG,EAAE,CAAC;YACnC,CAAC;YAAC,MAAM,CAAC;gBACP,gEAAgE;YAClE,CAAC;QACH,CAAC;QAED,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IACxB,CAAC;IAED,OAAO;QACL,OAAO;QACP,SAAS,EAAE,QAAQ,CAAC,WAAW,CAAC,CAAC,CAAC,QAAQ,CAAC,qBAAqB,CAAC,CAAC,CAAC,SAAS;KAC7E,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"list.test.d.ts","sourceRoot":"","sources":["../../src/sources/list.test.ts"],"names":[],"mappings":"AAAA;;;;;GAKG"}
|
|
@@ -0,0 +1,198 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Unit tests for sources/list.ts.
|
|
3
|
+
*
|
|
4
|
+
* Uses an in-memory S3Client stub installed via the internal `_setSourcesS3Factory`
|
|
5
|
+
* hook so we don't have to stand up vi.mock for @aws-sdk/client-s3.
|
|
6
|
+
*/
|
|
7
|
+
import { describe, it, expect, beforeEach, afterEach } from "vitest";
|
|
8
|
+
import { GetObjectCommand, ListObjectsV2Command, } from "@aws-sdk/client-s3";
|
|
9
|
+
import { listSources } from "./list.js";
|
|
10
|
+
import { _setSourcesS3Factory, _resetSourcesS3Factory, } from "./internals.js";
|
|
11
|
+
import { InvalidSourceChannelError } from "../schemas/source-channels.js";
|
|
12
|
+
// ---------------------------------------------------------------------------
|
|
13
|
+
// Fixtures + helpers
|
|
14
|
+
// ---------------------------------------------------------------------------
|
|
15
|
+
const ENTITY = {
|
|
16
|
+
uid: "cmp_test_001",
|
|
17
|
+
slug: "test",
|
|
18
|
+
bucketName: "hq-vault-test",
|
|
19
|
+
region: "us-east-1",
|
|
20
|
+
credentials: {
|
|
21
|
+
accessKeyId: "ASIATEST",
|
|
22
|
+
secretAccessKey: "secret",
|
|
23
|
+
sessionToken: "session",
|
|
24
|
+
},
|
|
25
|
+
expiresAt: new Date(Date.now() + 3600_000).toISOString(),
|
|
26
|
+
};
|
|
27
|
+
const MEETING_MD = `---
|
|
28
|
+
source_id: abc
|
|
29
|
+
source_type: meeting
|
|
30
|
+
channel: meeting
|
|
31
|
+
title: Hello
|
|
32
|
+
---
|
|
33
|
+
|
|
34
|
+
Body text.
|
|
35
|
+
`;
|
|
36
|
+
/** Build an in-memory S3 stub. Pagination is keyed by ContinuationToken (string offset). */
|
|
37
|
+
function buildStub(objects, observed = { commands: [] }) {
|
|
38
|
+
async function send(command) {
|
|
39
|
+
observed.commands.push(command);
|
|
40
|
+
if (command instanceof ListObjectsV2Command) {
|
|
41
|
+
const { Prefix, MaxKeys, ContinuationToken } = command.input;
|
|
42
|
+
const filtered = objects.filter((o) => !Prefix || o.key.startsWith(Prefix));
|
|
43
|
+
const offset = ContinuationToken ? parseInt(ContinuationToken, 10) : 0;
|
|
44
|
+
const max = MaxKeys ?? 1000;
|
|
45
|
+
const page = filtered.slice(offset, offset + max);
|
|
46
|
+
const nextOffset = offset + page.length;
|
|
47
|
+
const truncated = nextOffset < filtered.length;
|
|
48
|
+
return {
|
|
49
|
+
Contents: page.map((o) => ({
|
|
50
|
+
Key: o.key,
|
|
51
|
+
Size: Buffer.byteLength(o.content, "utf-8"),
|
|
52
|
+
LastModified: o.lastModified,
|
|
53
|
+
ETag: '"mock"',
|
|
54
|
+
})),
|
|
55
|
+
IsTruncated: truncated,
|
|
56
|
+
NextContinuationToken: truncated ? String(nextOffset) : undefined,
|
|
57
|
+
$metadata: {},
|
|
58
|
+
};
|
|
59
|
+
}
|
|
60
|
+
if (command instanceof GetObjectCommand) {
|
|
61
|
+
const { Key } = command.input;
|
|
62
|
+
const found = objects.find((o) => o.key === Key);
|
|
63
|
+
if (!found) {
|
|
64
|
+
const err = new Error(`NoSuchKey: ${Key}`);
|
|
65
|
+
err.name = "NoSuchKey";
|
|
66
|
+
throw err;
|
|
67
|
+
}
|
|
68
|
+
const content = found.content;
|
|
69
|
+
async function* stream() {
|
|
70
|
+
yield Buffer.from(content, "utf-8");
|
|
71
|
+
}
|
|
72
|
+
return { Body: stream(), $metadata: {} };
|
|
73
|
+
}
|
|
74
|
+
throw new Error(`unhandled command: ${command.constructor?.name}`);
|
|
75
|
+
}
|
|
76
|
+
return { send };
|
|
77
|
+
}
|
|
78
|
+
function installStub(objects) {
|
|
79
|
+
const observed = { commands: [] };
|
|
80
|
+
const stub = buildStub(objects, observed);
|
|
81
|
+
_setSourcesS3Factory(() => stub);
|
|
82
|
+
return observed;
|
|
83
|
+
}
|
|
84
|
+
beforeEach(() => {
|
|
85
|
+
_resetSourcesS3Factory();
|
|
86
|
+
});
|
|
87
|
+
afterEach(() => {
|
|
88
|
+
_resetSourcesS3Factory();
|
|
89
|
+
});
|
|
90
|
+
// ---------------------------------------------------------------------------
|
|
91
|
+
// Tests
|
|
92
|
+
// ---------------------------------------------------------------------------
|
|
93
|
+
describe("listSources", () => {
|
|
94
|
+
it("happy path: lists one source and skips its .raw.json sibling", async () => {
|
|
95
|
+
installStub([
|
|
96
|
+
{
|
|
97
|
+
key: "sources/meeting/abc.md",
|
|
98
|
+
content: MEETING_MD,
|
|
99
|
+
lastModified: new Date("2026-03-15T14:00:00Z"),
|
|
100
|
+
},
|
|
101
|
+
{
|
|
102
|
+
key: "sources/meeting/abc.raw.json",
|
|
103
|
+
content: '{"raw": true}',
|
|
104
|
+
lastModified: new Date("2026-03-15T14:00:00Z"),
|
|
105
|
+
},
|
|
106
|
+
]);
|
|
107
|
+
const result = await listSources({ entity: ENTITY, channel: "meeting" });
|
|
108
|
+
expect(result.entries).toHaveLength(1);
|
|
109
|
+
expect(result.entries[0].sourceId).toBe("abc");
|
|
110
|
+
expect(result.entries[0].channel).toBe("meeting");
|
|
111
|
+
expect(result.entries[0].key).toBe("sources/meeting/abc.md");
|
|
112
|
+
expect(result.entries[0].size).toBeGreaterThan(0);
|
|
113
|
+
expect(result.entries[0].frontmatter).toBeUndefined();
|
|
114
|
+
expect(result.nextToken).toBeUndefined();
|
|
115
|
+
});
|
|
116
|
+
it("pagination: returns nextToken and accepts it on the next call", async () => {
|
|
117
|
+
const objects = Array.from({ length: 5 }, (_, i) => ({
|
|
118
|
+
key: `sources/meeting/m${i}.md`,
|
|
119
|
+
content: MEETING_MD,
|
|
120
|
+
lastModified: new Date("2026-03-15T14:00:00Z"),
|
|
121
|
+
}));
|
|
122
|
+
installStub(objects);
|
|
123
|
+
const page1 = await listSources({
|
|
124
|
+
entity: ENTITY,
|
|
125
|
+
channel: "meeting",
|
|
126
|
+
limit: 2,
|
|
127
|
+
});
|
|
128
|
+
expect(page1.entries).toHaveLength(2);
|
|
129
|
+
expect(page1.entries.map((e) => e.sourceId)).toEqual(["m0", "m1"]);
|
|
130
|
+
expect(page1.nextToken).toBe("2");
|
|
131
|
+
const page2 = await listSources({
|
|
132
|
+
entity: ENTITY,
|
|
133
|
+
channel: "meeting",
|
|
134
|
+
limit: 2,
|
|
135
|
+
continuationToken: page1.nextToken,
|
|
136
|
+
});
|
|
137
|
+
expect(page2.entries.map((e) => e.sourceId)).toEqual(["m2", "m3"]);
|
|
138
|
+
expect(page2.nextToken).toBe("4");
|
|
139
|
+
const page3 = await listSources({
|
|
140
|
+
entity: ENTITY,
|
|
141
|
+
channel: "meeting",
|
|
142
|
+
limit: 2,
|
|
143
|
+
continuationToken: page2.nextToken,
|
|
144
|
+
});
|
|
145
|
+
expect(page3.entries.map((e) => e.sourceId)).toEqual(["m4"]);
|
|
146
|
+
expect(page3.nextToken).toBeUndefined();
|
|
147
|
+
});
|
|
148
|
+
it("includeFrontmatter:true fetches each object and parses YAML", async () => {
|
|
149
|
+
const observed = installStub([
|
|
150
|
+
{
|
|
151
|
+
key: "sources/meeting/abc.md",
|
|
152
|
+
content: MEETING_MD,
|
|
153
|
+
lastModified: new Date("2026-03-15T14:00:00Z"),
|
|
154
|
+
},
|
|
155
|
+
]);
|
|
156
|
+
const result = await listSources({
|
|
157
|
+
entity: ENTITY,
|
|
158
|
+
channel: "meeting",
|
|
159
|
+
includeFrontmatter: true,
|
|
160
|
+
});
|
|
161
|
+
expect(result.entries).toHaveLength(1);
|
|
162
|
+
expect(result.entries[0].frontmatter).toEqual({
|
|
163
|
+
source_id: "abc",
|
|
164
|
+
source_type: "meeting",
|
|
165
|
+
channel: "meeting",
|
|
166
|
+
title: "Hello",
|
|
167
|
+
});
|
|
168
|
+
// 1 list + 1 get for the single entry.
|
|
169
|
+
expect(observed.commands.filter((c) => c instanceof GetObjectCommand)).toHaveLength(1);
|
|
170
|
+
});
|
|
171
|
+
it("includeFrontmatter:false (default) does not perform GETs", async () => {
|
|
172
|
+
const observed = installStub([
|
|
173
|
+
{
|
|
174
|
+
key: "sources/meeting/abc.md",
|
|
175
|
+
content: MEETING_MD,
|
|
176
|
+
lastModified: new Date(),
|
|
177
|
+
},
|
|
178
|
+
]);
|
|
179
|
+
await listSources({ entity: ENTITY, channel: "meeting" });
|
|
180
|
+
expect(observed.commands.filter((c) => c instanceof GetObjectCommand)).toHaveLength(0);
|
|
181
|
+
});
|
|
182
|
+
it("uses the correct prefix per channel", async () => {
|
|
183
|
+
const observed = installStub([]);
|
|
184
|
+
await listSources({ entity: ENTITY, channel: "slack" });
|
|
185
|
+
const listCmd = observed.commands[0];
|
|
186
|
+
expect(listCmd.input.Prefix).toBe("sources/slack/");
|
|
187
|
+
});
|
|
188
|
+
it("rejects an invalid channel via assertSourceChannel BEFORE any S3 call", async () => {
|
|
189
|
+
const observed = installStub([]);
|
|
190
|
+
await expect(listSources({
|
|
191
|
+
entity: ENTITY,
|
|
192
|
+
// @ts-expect-error — invalid channel intentional for negative test
|
|
193
|
+
channel: "pigeon",
|
|
194
|
+
})).rejects.toBeInstanceOf(InvalidSourceChannelError);
|
|
195
|
+
expect(observed.commands).toHaveLength(0);
|
|
196
|
+
});
|
|
197
|
+
});
|
|
198
|
+
//# sourceMappingURL=list.test.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"list.test.js","sourceRoot":"","sources":["../../src/sources/list.test.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,QAAQ,CAAC;AACrE,OAAO,EACL,gBAAgB,EAChB,oBAAoB,GAErB,MAAM,oBAAoB,CAAC;AAC5B,OAAO,EAAE,WAAW,EAAE,MAAM,WAAW,CAAC;AACxC,OAAO,EACL,oBAAoB,EACpB,sBAAsB,GACvB,MAAM,gBAAgB,CAAC;AAExB,OAAO,EAAE,yBAAyB,EAAE,MAAM,+BAA+B,CAAC;AAE1E,8EAA8E;AAC9E,qBAAqB;AACrB,8EAA8E;AAE9E,MAAM,MAAM,GAAkB;IAC5B,GAAG,EAAE,cAAc;IACnB,IAAI,EAAE,MAAM;IACZ,UAAU,EAAE,eAAe;IAC3B,MAAM,EAAE,WAAW;IACnB,WAAW,EAAE;QACX,WAAW,EAAE,UAAU;QACvB,eAAe,EAAE,QAAQ;QACzB,YAAY,EAAE,SAAS;KACxB;IACD,SAAS,EAAE,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,QAAQ,CAAC,CAAC,WAAW,EAAE;CACzD,CAAC;AAEF,MAAM,UAAU,GAAG;;;;;;;;CAQlB,CAAC;AAQF,4FAA4F;AAC5F,SAAS,SAAS,CAChB,OAAuB,EACvB,WAAoC,EAAE,QAAQ,EAAE,EAAE,EAAE;IAEpD,KAAK,UAAU,IAAI,CAAC,OAAgB;QAClC,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QAEhC,IAAI,OAAO,YAAY,oBAAoB,EAAE,CAAC;YAC5C,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,iBAAiB,EAAE,GAAG,OAAO,CAAC,KAItD,CAAC;YACF,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,GAAG,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC;YAC5E,MAAM,MAAM,GAAG,iBAAiB,CAAC,CAAC,CAAC,QAAQ,CAAC,iBAAiB,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YACvE,MAAM,GAAG,GAAG,OAAO,IAAI,IAAI,CAAC;YAC5B,MAAM,IAAI,GAAG,QAAQ,CAAC,KAAK,CAAC,MAAM,EAAE,MAAM,GAAG,GAAG,CAAC,CAAC;YAClD,MAAM,UAAU,GAAG,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;YACxC,MAAM,SAAS,GAAG,UAAU,GAAG,QAAQ,CAAC,MAAM,CAAC;YAC/C,OAAO;gBACL,QAAQ,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;oBACzB,GAAG,EAAE,CAAC,CAAC,GAAG;oBACV,IAAI,EAAE,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC,OAAO,EAAE,OAAO,CAAC;oBAC3C,YAAY,EAAE,CAAC,CAAC,YAAY;oBAC5B,IAAI,EAAE,QAAQ;iBACf,CAAC,CAAC;gBACH,WAAW,EAAE,SAAS;gBACtB,qBAAqB,EAAE,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,SAAS;gBACjE,SAAS,EAAE,EAAE;aACd,CAAC;QACJ,CAAC;QAED,IAAI,OAAO,YAAY,gBAAgB,EAAE,CAAC;YACxC,MAAM,EAAE,GAAG,EAAE,GAAG,OAAO,CAAC,KAAwB,CAAC;YACjD,MAAM,KAAK,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,KAAK,GAAG,CAAC,CAAC;YACjD,IAAI,CAAC,KAAK,EAAE,CAAC;gBACX,MAAM,GAAG,GAAG,IAAI,KAAK,CAAC,cAAc,GAAG,EAAE,CAAC,CAAC;gBAC1C,GAAgC,CAAC,IAAI,GAAG,WAAW,CAAC;gBACrD,MAAM,GAAG,CAAC;YACZ,CAAC;YACD,MAAM,OAAO,GAAG,KAAK,CAAC,OAAO,CAAC;YAC9B,KAAK,SAAS,CAAC,CAAC,MAAM;gBACpB,MAAM,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;YACtC,CAAC;YACD,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,EAAE,SAAS,EAAE,EAAE,EAAE,CAAC;QAC3C,CAAC;QAED,MAAM,IAAI,KAAK,CAAC,sBAAuB,OAA+C,CAAC,WAAW,EAAE,IAAI,EAAE,CAAC,CAAC;IAC9G,CAAC;IACD,OAAO,EAAE,IAAI,EAAyB,CAAC;AACzC,CAAC;AAED,SAAS,WAAW,CAAC,OAAuB;IAC1C,MAAM,QAAQ,GAAG,EAAE,QAAQ,EAAE,EAAe,EAAE,CAAC;IAC/C,MAAM,IAAI,GAAG,SAAS,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;IAC1C,oBAAoB,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,CAAC;IACjC,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,UAAU,CAAC,GAAG,EAAE;IACd,sBAAsB,EAAE,CAAC;AAC3B,CAAC,CAAC,CAAC;AAEH,SAAS,CAAC,GAAG,EAAE;IACb,sBAAsB,EAAE,CAAC;AAC3B,CAAC,CAAC,CAAC;AAEH,8EAA8E;AAC9E,QAAQ;AACR,8EAA8E;AAE9E,QAAQ,CAAC,aAAa,EAAE,GAAG,EAAE;IAC3B,EAAE,CAAC,8DAA8D,EAAE,KAAK,IAAI,EAAE;QAC5E,WAAW,CAAC;YACV;gBACE,GAAG,EAAE,wBAAwB;gBAC7B,OAAO,EAAE,UAAU;gBACnB,YAAY,EAAE,IAAI,IAAI,CAAC,sBAAsB,CAAC;aAC/C;YACD;gBACE,GAAG,EAAE,8BAA8B;gBACnC,OAAO,EAAE,eAAe;gBACxB,YAAY,EAAE,IAAI,IAAI,CAAC,sBAAsB,CAAC;aAC/C;SACF,CAAC,CAAC;QAEH,MAAM,MAAM,GAAG,MAAM,WAAW,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,SAAS,EAAE,CAAC,CAAC;QAEzE,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QACvC,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAC/C,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QAClD,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,wBAAwB,CAAC,CAAC;QAC7D,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC;QAClD,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,aAAa,EAAE,CAAC;QACtD,MAAM,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,aAAa,EAAE,CAAC;IAC3C,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,+DAA+D,EAAE,KAAK,IAAI,EAAE;QAC7E,MAAM,OAAO,GAAmB,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;YACnE,GAAG,EAAE,oBAAoB,CAAC,KAAK;YAC/B,OAAO,EAAE,UAAU;YACnB,YAAY,EAAE,IAAI,IAAI,CAAC,sBAAsB,CAAC;SAC/C,CAAC,CAAC,CAAC;QACJ,WAAW,CAAC,OAAO,CAAC,CAAC;QAErB,MAAM,KAAK,GAAG,MAAM,WAAW,CAAC;YAC9B,MAAM,EAAE,MAAM;YACd,OAAO,EAAE,SAAS;YAClB,KAAK,EAAE,CAAC;SACT,CAAC,CAAC;QACH,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QACtC,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC;QACnE,MAAM,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAElC,MAAM,KAAK,GAAG,MAAM,WAAW,CAAC;YAC9B,MAAM,EAAE,MAAM;YACd,OAAO,EAAE,SAAS;YAClB,KAAK,EAAE,CAAC;YACR,iBAAiB,EAAE,KAAK,CAAC,SAAS;SACnC,CAAC,CAAC;QACH,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC;QACnE,MAAM,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAElC,MAAM,KAAK,GAAG,MAAM,WAAW,CAAC;YAC9B,MAAM,EAAE,MAAM;YACd,OAAO,EAAE,SAAS;YAClB,KAAK,EAAE,CAAC;YACR,iBAAiB,EAAE,KAAK,CAAC,SAAS;SACnC,CAAC,CAAC;QACH,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;QAC7D,MAAM,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,aAAa,EAAE,CAAC;IAC1C,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,6DAA6D,EAAE,KAAK,IAAI,EAAE;QAC3E,MAAM,QAAQ,GAAG,WAAW,CAAC;YAC3B;gBACE,GAAG,EAAE,wBAAwB;gBAC7B,OAAO,EAAE,UAAU;gBACnB,YAAY,EAAE,IAAI,IAAI,CAAC,sBAAsB,CAAC;aAC/C;SACF,CAAC,CAAC;QAEH,MAAM,MAAM,GAAG,MAAM,WAAW,CAAC;YAC/B,MAAM,EAAE,MAAM;YACd,OAAO,EAAE,SAAS;YAClB,kBAAkB,EAAE,IAAI;SACzB,CAAC,CAAC;QAEH,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QACvC,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,OAAO,CAAC;YAC5C,SAAS,EAAE,KAAK;YAChB,WAAW,EAAE,SAAS;YACtB,OAAO,EAAE,SAAS;YAClB,KAAK,EAAE,OAAO;SACf,CAAC,CAAC;QACH,uCAAuC;QACvC,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,YAAY,gBAAgB,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;IACzF,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,0DAA0D,EAAE,KAAK,IAAI,EAAE;QACxE,MAAM,QAAQ,GAAG,WAAW,CAAC;YAC3B;gBACE,GAAG,EAAE,wBAAwB;gBAC7B,OAAO,EAAE,UAAU;gBACnB,YAAY,EAAE,IAAI,IAAI,EAAE;aACzB;SACF,CAAC,CAAC;QAEH,MAAM,WAAW,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,SAAS,EAAE,CAAC,CAAC;QAE1D,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,YAAY,gBAAgB,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;IACzF,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,qCAAqC,EAAE,KAAK,IAAI,EAAE;QACnD,MAAM,QAAQ,GAAG,WAAW,CAAC,EAAE,CAAC,CAAC;QACjC,MAAM,WAAW,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC,CAAC;QACxD,MAAM,OAAO,GAAG,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAyB,CAAC;QAC7D,MAAM,CAAE,OAAO,CAAC,KAA6B,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;IAC/E,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,uEAAuE,EAAE,KAAK,IAAI,EAAE;QACrF,MAAM,QAAQ,GAAG,WAAW,CAAC,EAAE,CAAC,CAAC;QACjC,MAAM,MAAM,CACV,WAAW,CAAC;YACV,MAAM,EAAE,MAAM;YACd,mEAAmE;YACnE,OAAO,EAAE,QAAQ;SAClB,CAAC,CACH,CAAC,OAAO,CAAC,cAAc,CAAC,yBAAyB,CAAC,CAAC;QACpD,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;IAC5C,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* YAML frontmatter parsing for source markdown documents.
|
|
3
|
+
*
|
|
4
|
+
* Format: a leading `---\n` block terminated by `\n---\n`, followed by the body.
|
|
5
|
+
* Tolerates LF and CRLF. Returns `{ frontmatter: null, body }` when no
|
|
6
|
+
* frontmatter block is present.
|
|
7
|
+
*/
|
|
8
|
+
export interface ParsedMarkdown {
|
|
9
|
+
frontmatter: Record<string, unknown> | null;
|
|
10
|
+
body: string;
|
|
11
|
+
}
|
|
12
|
+
export declare function parseMarkdown(content: string): ParsedMarkdown;
|
|
13
|
+
/**
|
|
14
|
+
* Convenience: parse only the frontmatter (used by listSources includeFrontmatter
|
|
15
|
+
* where we don't need the body).
|
|
16
|
+
*/
|
|
17
|
+
export declare function parseFrontmatter(content: string): Record<string, unknown> | null;
|
|
18
|
+
//# sourceMappingURL=parse.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"parse.d.ts","sourceRoot":"","sources":["../../src/sources/parse.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAIH,MAAM,WAAW,cAAc;IAC7B,WAAW,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,CAAC;IAC5C,IAAI,EAAE,MAAM,CAAC;CACd;AAID,wBAAgB,aAAa,CAAC,OAAO,EAAE,MAAM,GAAG,cAAc,CAiB7D;AAED;;;GAGG;AACH,wBAAgB,gBAAgB,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,CAEhF"}
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* YAML frontmatter parsing for source markdown documents.
|
|
3
|
+
*
|
|
4
|
+
* Format: a leading `---\n` block terminated by `\n---\n`, followed by the body.
|
|
5
|
+
* Tolerates LF and CRLF. Returns `{ frontmatter: null, body }` when no
|
|
6
|
+
* frontmatter block is present.
|
|
7
|
+
*/
|
|
8
|
+
import { load as yamlLoad } from "js-yaml";
|
|
9
|
+
const FRONTMATTER_RE = /^---\r?\n([\s\S]*?)\r?\n---\r?\n?([\s\S]*)$/;
|
|
10
|
+
export function parseMarkdown(content) {
|
|
11
|
+
const match = FRONTMATTER_RE.exec(content);
|
|
12
|
+
if (!match) {
|
|
13
|
+
return { frontmatter: null, body: content };
|
|
14
|
+
}
|
|
15
|
+
const [, yamlBlock, body] = match;
|
|
16
|
+
let frontmatter;
|
|
17
|
+
try {
|
|
18
|
+
const loaded = yamlLoad(yamlBlock);
|
|
19
|
+
frontmatter = loaded && typeof loaded === "object" && !Array.isArray(loaded)
|
|
20
|
+
? loaded
|
|
21
|
+
: null;
|
|
22
|
+
}
|
|
23
|
+
catch {
|
|
24
|
+
frontmatter = null;
|
|
25
|
+
}
|
|
26
|
+
return { frontmatter, body };
|
|
27
|
+
}
|
|
28
|
+
/**
|
|
29
|
+
* Convenience: parse only the frontmatter (used by listSources includeFrontmatter
|
|
30
|
+
* where we don't need the body).
|
|
31
|
+
*/
|
|
32
|
+
export function parseFrontmatter(content) {
|
|
33
|
+
return parseMarkdown(content).frontmatter;
|
|
34
|
+
}
|
|
35
|
+
//# sourceMappingURL=parse.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"parse.js","sourceRoot":"","sources":["../../src/sources/parse.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,EAAE,IAAI,IAAI,QAAQ,EAAE,MAAM,SAAS,CAAC;AAO3C,MAAM,cAAc,GAAG,6CAA6C,CAAC;AAErE,MAAM,UAAU,aAAa,CAAC,OAAe;IAC3C,MAAM,KAAK,GAAG,cAAc,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IAC3C,IAAI,CAAC,KAAK,EAAE,CAAC;QACX,OAAO,EAAE,WAAW,EAAE,IAAI,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC;IAC9C,CAAC;IAED,MAAM,CAAC,EAAE,SAAS,EAAE,IAAI,CAAC,GAAG,KAAK,CAAC;IAClC,IAAI,WAA2C,CAAC;IAChD,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,QAAQ,CAAC,SAAS,CAAC,CAAC;QACnC,WAAW,GAAG,MAAM,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC;YAC1E,CAAC,CAAE,MAAkC;YACrC,CAAC,CAAC,IAAI,CAAC;IACX,CAAC;IAAC,MAAM,CAAC;QACP,WAAW,GAAG,IAAI,CAAC;IACrB,CAAC;IACD,OAAO,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC;AAC/B,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,gBAAgB,CAAC,OAAe;IAC9C,OAAO,aAAa,CAAC,OAAO,CAAC,CAAC,WAAW,CAAC;AAC5C,CAAC"}
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Public types for the sources read surface.
|
|
3
|
+
*
|
|
4
|
+
* Mirrors the sources-pipeline write path so a meeting source written by
|
|
5
|
+
* hq-pro is readable through this layer without translation.
|
|
6
|
+
*/
|
|
7
|
+
import type { SourceChannel } from "../schemas/source-channels.js";
|
|
8
|
+
/**
|
|
9
|
+
* Lightweight summary returned by listSources(). `frontmatter` is only
|
|
10
|
+
* populated when the caller passes `includeFrontmatter: true`.
|
|
11
|
+
*/
|
|
12
|
+
export interface SourceSummary {
|
|
13
|
+
/** Stable ID derived from the object key (filename minus `.md`/`.raw.json`). */
|
|
14
|
+
sourceId: string;
|
|
15
|
+
channel: SourceChannel;
|
|
16
|
+
/** Full S3 object key (e.g. "sources/meeting/abc.md"). */
|
|
17
|
+
key: string;
|
|
18
|
+
lastModified: Date;
|
|
19
|
+
size: number;
|
|
20
|
+
/** Parsed YAML frontmatter — only present when listSources(includeFrontmatter:true). */
|
|
21
|
+
frontmatter?: Record<string, unknown>;
|
|
22
|
+
}
|
|
23
|
+
/**
|
|
24
|
+
* Full source document returned by getSource(). `frontmatter` is `null` if
|
|
25
|
+
* the document was malformed (no `---` delimiters) — body still contains the
|
|
26
|
+
* raw payload in that case so callers can fall back.
|
|
27
|
+
*/
|
|
28
|
+
export interface SourceDocument {
|
|
29
|
+
key: string;
|
|
30
|
+
/** Parsed YAML frontmatter, or `null` if the document had no frontmatter block. */
|
|
31
|
+
frontmatter: Record<string, unknown> | null;
|
|
32
|
+
/** Markdown body (everything after the closing `---`). */
|
|
33
|
+
body: string;
|
|
34
|
+
/** Sibling `.raw.json` payload, only present when getSource(includeRaw:true). */
|
|
35
|
+
raw?: unknown;
|
|
36
|
+
}
|
|
37
|
+
/**
|
|
38
|
+
* Listing options for listSources().
|
|
39
|
+
*/
|
|
40
|
+
export interface ListSourcesOptions {
|
|
41
|
+
entity: import("../types.js").EntityContext;
|
|
42
|
+
channel: SourceChannel;
|
|
43
|
+
/** Max keys per page; defaults to S3 default (1000). */
|
|
44
|
+
limit?: number;
|
|
45
|
+
/** Opaque continuation token returned by a prior page. */
|
|
46
|
+
continuationToken?: string;
|
|
47
|
+
/** When true, fetches+parses each entry's frontmatter (extra GETs). */
|
|
48
|
+
includeFrontmatter?: boolean;
|
|
49
|
+
}
|
|
50
|
+
export interface ListSourcesResult {
|
|
51
|
+
entries: SourceSummary[];
|
|
52
|
+
/** Present when the result was truncated; pass back as continuationToken. */
|
|
53
|
+
nextToken?: string;
|
|
54
|
+
}
|
|
55
|
+
export interface GetSourceOptions {
|
|
56
|
+
entity: import("../types.js").EntityContext;
|
|
57
|
+
channel: SourceChannel;
|
|
58
|
+
sourceId: string;
|
|
59
|
+
/** When true, also fetches the `.raw.json` sibling. */
|
|
60
|
+
includeRaw?: boolean;
|
|
61
|
+
}
|
|
62
|
+
//# sourceMappingURL=types.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/sources/types.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,+BAA+B,CAAC;AAEnE;;;GAGG;AACH,MAAM,WAAW,aAAa;IAC5B,gFAAgF;IAChF,QAAQ,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,aAAa,CAAC;IACvB,0DAA0D;IAC1D,GAAG,EAAE,MAAM,CAAC;IACZ,YAAY,EAAE,IAAI,CAAC;IACnB,IAAI,EAAE,MAAM,CAAC;IACb,wFAAwF;IACxF,WAAW,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACvC;AAED;;;;GAIG;AACH,MAAM,WAAW,cAAc;IAC7B,GAAG,EAAE,MAAM,CAAC;IACZ,mFAAmF;IACnF,WAAW,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,CAAC;IAC5C,0DAA0D;IAC1D,IAAI,EAAE,MAAM,CAAC;IACb,iFAAiF;IACjF,GAAG,CAAC,EAAE,OAAO,CAAC;CACf;AAED;;GAEG;AACH,MAAM,WAAW,kBAAkB;IACjC,MAAM,EAAE,OAAO,aAAa,EAAE,aAAa,CAAC;IAC5C,OAAO,EAAE,aAAa,CAAC;IACvB,wDAAwD;IACxD,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,0DAA0D;IAC1D,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,uEAAuE;IACvE,kBAAkB,CAAC,EAAE,OAAO,CAAC;CAC9B;AAED,MAAM,WAAW,iBAAiB;IAChC,OAAO,EAAE,aAAa,EAAE,CAAC;IACzB,6EAA6E;IAC7E,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,gBAAgB;IAC/B,MAAM,EAAE,OAAO,aAAa,EAAE,aAAa,CAAC;IAC5C,OAAO,EAAE,aAAa,CAAC;IACvB,QAAQ,EAAE,MAAM,CAAC;IACjB,uDAAuD;IACvD,UAAU,CAAC,EAAE,OAAO,CAAC;CACtB"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/sources/types.ts"],"names":[],"mappings":"AAAA;;;;;GAKG"}
|
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Usage telemetry collector — TypeScript port of the Tauri Rust collector that
|
|
3
|
+
* used to live at `hq-workspace/apps/hq-sync/src-tauri/src/commands/telemetry.rs`.
|
|
4
|
+
*
|
|
5
|
+
* Why it moved: the Rust copy only ran inside the macOS menubar app. By moving
|
|
6
|
+
* the logic into `@indigoai-us/hq-cloud`, every consumer of the package
|
|
7
|
+
* (`hq-sync-runner`, `hq-cli`, mobile wrappers) emits telemetry uniformly.
|
|
8
|
+
*
|
|
9
|
+
* What it does: after each successful sync (`all-complete` arm of
|
|
10
|
+
* `bin/sync-runner.ts`), walks `~/.claude/projects/**\/*.jsonl`, diffs each
|
|
11
|
+
* file against a persisted byte-offset cursor at `~/.hq/telemetry-cursor.json`,
|
|
12
|
+
* sanitizes new rows through a tight allowlist that matches the server's
|
|
13
|
+
* KEEP_FIELDS set in `apps/hq-pro/src/vault-service/handlers/usage.ts`,
|
|
14
|
+
* batches into ≤1 MiB POST bodies, and ships them to `/v1/usage`.
|
|
15
|
+
*
|
|
16
|
+
* Trust model: the caller's `personUid` is resolved on the server from the
|
|
17
|
+
* Cognito JWT — never from the body. `sanitizeRow` strips prompt bodies,
|
|
18
|
+
* thinking content, tool inputs/outputs, and any nested `message` object so
|
|
19
|
+
* the wire payload contains only token-accounting fields.
|
|
20
|
+
*
|
|
21
|
+
* Errors are swallowed by design — telemetry must never abort or delay a
|
|
22
|
+
* sync. The cursor is only advanced for batches the server 2xx'd, so a
|
|
23
|
+
* transient outage retries automatically on the next sync.
|
|
24
|
+
*/
|
|
25
|
+
import type { TelemetryOptInResponse, UsageBatch, UsageIngestResult } from "./vault-client.js";
|
|
26
|
+
/**
|
|
27
|
+
* Minimal subset of `VaultClient` the collector needs. Declared as an
|
|
28
|
+
* interface so tests can inject a stub without spinning up a fetch mock.
|
|
29
|
+
* The real `VaultClient` from `./vault-client.js` satisfies this structurally.
|
|
30
|
+
*/
|
|
31
|
+
export interface TelemetryClientSurface {
|
|
32
|
+
getTelemetryOptIn(): Promise<TelemetryOptInResponse>;
|
|
33
|
+
postUsage(batch: UsageBatch): Promise<UsageIngestResult>;
|
|
34
|
+
}
|
|
35
|
+
export interface CollectTelemetryOptions {
|
|
36
|
+
client: TelemetryClientSurface;
|
|
37
|
+
/** Stable per-machine id. The Tauri menubar reads this from `~/.hq/menubar.json`; the runner can pass it through or generate one once and cache. */
|
|
38
|
+
machineId: string;
|
|
39
|
+
/** Version of the wrapping caller (menubar app, CLI, etc.). Reaches CloudWatch metrics as the `installerVersion` dimension. */
|
|
40
|
+
installerVersion: string;
|
|
41
|
+
/** Override `~/.claude/projects` for tests. */
|
|
42
|
+
claudeProjectsRoot?: string;
|
|
43
|
+
/** Override `~/.hq/telemetry-cursor.json` for tests. */
|
|
44
|
+
cursorPath?: string;
|
|
45
|
+
/** Override `~/.hq/menubar.json` (the offline opt-in fallback) for tests. */
|
|
46
|
+
menubarPath?: string;
|
|
47
|
+
/** Diagnostic sink. No-op by default. */
|
|
48
|
+
log?: (msg: string) => void;
|
|
49
|
+
}
|
|
50
|
+
export interface CollectTelemetryResult {
|
|
51
|
+
/** Whether the opt-in check resolved to true (either server-side or via the menubar fallback). When false, nothing else ran. */
|
|
52
|
+
enabled: boolean;
|
|
53
|
+
/** Source for the `enabled` decision — useful for diagnosing missing-events reports. */
|
|
54
|
+
optInSource: "server" | "menubar-fallback" | "skipped";
|
|
55
|
+
/** How many `.jsonl` files we considered (before the cursor diff). */
|
|
56
|
+
filesScanned: number;
|
|
57
|
+
/** Total events successfully POSTed across all batches. */
|
|
58
|
+
eventsSent: number;
|
|
59
|
+
/** Number of `POST /v1/usage` requests made. */
|
|
60
|
+
batchesSent: number;
|
|
61
|
+
}
|
|
62
|
+
/**
|
|
63
|
+
* Build an outgoing event row matching the server's KEEP allowlist.
|
|
64
|
+
*
|
|
65
|
+
* Two transforms:
|
|
66
|
+
* 1. Top-level fields are copied straight through (string identity).
|
|
67
|
+
* 2. `message.model` and `message.usage.{input_tokens, output_tokens,
|
|
68
|
+
* cache_creation_input_tokens, cache_read_input_tokens}` are promoted to
|
|
69
|
+
* camelCase top-level fields. The original `message` object — which
|
|
70
|
+
* carries prompt/response text, thinking, and tool data — is dropped.
|
|
71
|
+
*
|
|
72
|
+
* Returns `null` when the input isn't an object. Empty results (e.g. a row
|
|
73
|
+
* with no recognised fields) are still returned as `{}` and emitted; the
|
|
74
|
+
* server accepts empty rows and they're useful as a "Claude Code was run at
|
|
75
|
+
* this time" heartbeat.
|
|
76
|
+
*/
|
|
77
|
+
export declare function sanitizeRow(row: unknown): Record<string, unknown> | null;
|
|
78
|
+
/**
|
|
79
|
+
* Scan, sanitize, and POST any new Claude Code session rows.
|
|
80
|
+
*
|
|
81
|
+
* Fire-and-forget from the caller's perspective: errors are caught internally
|
|
82
|
+
* and surfaced only via `log`. The returned summary lets observers (e.g.
|
|
83
|
+
* sync-runner) decide whether to record a "telemetry attempted" breadcrumb,
|
|
84
|
+
* but no consumer is expected to react to it.
|
|
85
|
+
*/
|
|
86
|
+
export declare function collectAndSendTelemetry(opts: CollectTelemetryOptions): Promise<CollectTelemetryResult>;
|
|
87
|
+
//# sourceMappingURL=telemetry.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"telemetry.d.ts","sourceRoot":"","sources":["../src/telemetry.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AAMH,OAAO,KAAK,EACV,sBAAsB,EACtB,UAAU,EACV,iBAAiB,EAClB,MAAM,mBAAmB,CAAC;AAI3B;;;;GAIG;AACH,MAAM,WAAW,sBAAsB;IACrC,iBAAiB,IAAI,OAAO,CAAC,sBAAsB,CAAC,CAAC;IACrD,SAAS,CAAC,KAAK,EAAE,UAAU,GAAG,OAAO,CAAC,iBAAiB,CAAC,CAAC;CAC1D;AAED,MAAM,WAAW,uBAAuB;IACtC,MAAM,EAAE,sBAAsB,CAAC;IAC/B,oJAAoJ;IACpJ,SAAS,EAAE,MAAM,CAAC;IAClB,+HAA+H;IAC/H,gBAAgB,EAAE,MAAM,CAAC;IACzB,+CAA+C;IAC/C,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,wDAAwD;IACxD,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,6EAA6E;IAC7E,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,yCAAyC;IACzC,GAAG,CAAC,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,IAAI,CAAC;CAC7B;AAED,MAAM,WAAW,sBAAsB;IACrC,gIAAgI;IAChI,OAAO,EAAE,OAAO,CAAC;IACjB,wFAAwF;IACxF,WAAW,EAAE,QAAQ,GAAG,kBAAkB,GAAG,SAAS,CAAC;IACvD,sEAAsE;IACtE,YAAY,EAAE,MAAM,CAAC;IACrB,2DAA2D;IAC3D,UAAU,EAAE,MAAM,CAAC;IACnB,gDAAgD;IAChD,WAAW,EAAE,MAAM,CAAC;CACrB;AAkED;;;;;;;;;;;;;;GAcG;AACH,wBAAgB,WAAW,CAAC,GAAG,EAAE,OAAO,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,CA8BxE;AA6DD;;;;;;;GAOG;AACH,wBAAsB,uBAAuB,CAC3C,IAAI,EAAE,uBAAuB,GAC5B,OAAO,CAAC,sBAAsB,CAAC,CAmLjC"}
|