@indigoai-us/hq-cloud 6.11.5 → 6.11.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin/sync-runner.d.ts +16 -16
- package/dist/bin/sync-runner.d.ts.map +1 -1
- package/dist/bin/sync-runner.js +51 -41
- package/dist/bin/sync-runner.js.map +1 -1
- package/dist/bin/sync-runner.test.js +108 -33
- package/dist/bin/sync-runner.test.js.map +1 -1
- package/dist/cli/share.d.ts +23 -0
- package/dist/cli/share.d.ts.map +1 -1
- package/dist/cli/share.js +54 -0
- package/dist/cli/share.js.map +1 -1
- package/dist/cli/share.test.js +142 -0
- package/dist/cli/share.test.js.map +1 -1
- package/dist/cli/sync.d.ts +16 -0
- package/dist/cli/sync.d.ts.map +1 -1
- package/dist/cli/sync.js +1 -62
- package/dist/cli/sync.js.map +1 -1
- package/dist/cli/tombstones.d.ts +43 -0
- package/dist/cli/tombstones.d.ts.map +1 -0
- package/dist/cli/tombstones.js +78 -0
- package/dist/cli/tombstones.js.map +1 -0
- package/dist/context.d.ts +6 -0
- package/dist/context.d.ts.map +1 -1
- package/dist/context.js +57 -17
- package/dist/context.js.map +1 -1
- package/dist/context.test.js +113 -1
- package/dist/context.test.js.map +1 -1
- package/dist/entity-resolver.test.js +3 -3
- package/dist/entity-resolver.test.js.map +1 -1
- package/dist/index.d.ts +1 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -1
- package/dist/index.js.map +1 -1
- package/dist/object-io.d.ts.map +1 -1
- package/dist/object-io.js +10 -0
- package/dist/object-io.js.map +1 -1
- package/dist/personal-vault.d.ts +36 -0
- package/dist/personal-vault.d.ts.map +1 -1
- package/dist/personal-vault.js +89 -1
- package/dist/personal-vault.js.map +1 -1
- package/dist/personal-vault.test.js +143 -1
- package/dist/personal-vault.test.js.map +1 -1
- package/dist/signals/get.d.ts.map +1 -1
- package/dist/signals/get.js +7 -11
- package/dist/signals/get.js.map +1 -1
- package/dist/signals/get.test.js +65 -1
- package/dist/signals/get.test.js.map +1 -1
- package/dist/signals/internals.d.ts +47 -3
- package/dist/signals/internals.d.ts.map +1 -1
- package/dist/signals/internals.js +110 -4
- package/dist/signals/internals.js.map +1 -1
- package/dist/signals/list.d.ts.map +1 -1
- package/dist/signals/list.js +16 -23
- package/dist/signals/list.js.map +1 -1
- package/dist/signals/list.test.js +84 -1
- package/dist/signals/list.test.js.map +1 -1
- package/dist/signals/types.d.ts +18 -1
- package/dist/signals/types.d.ts.map +1 -1
- package/dist/sources/get.d.ts.map +1 -1
- package/dist/sources/get.js +10 -22
- package/dist/sources/get.js.map +1 -1
- package/dist/sources/get.test.js +85 -1
- package/dist/sources/get.test.js.map +1 -1
- package/dist/sources/internals.d.ts +50 -3
- package/dist/sources/internals.d.ts.map +1 -1
- package/dist/sources/internals.js +113 -4
- package/dist/sources/internals.js.map +1 -1
- package/dist/sources/list.d.ts.map +1 -1
- package/dist/sources/list.js +16 -23
- package/dist/sources/list.js.map +1 -1
- package/dist/sources/list.test.js +101 -1
- package/dist/sources/list.test.js.map +1 -1
- package/dist/sources/types.d.ts +18 -1
- package/dist/sources/types.d.ts.map +1 -1
- package/dist/sync/event-sync.d.ts +6 -7
- package/dist/sync/event-sync.d.ts.map +1 -1
- package/dist/sync/event-sync.js +6 -7
- package/dist/sync/event-sync.js.map +1 -1
- package/dist/types.d.ts +33 -3
- package/dist/types.d.ts.map +1 -1
- package/dist/version.d.ts +14 -0
- package/dist/version.d.ts.map +1 -0
- package/dist/version.js +20 -0
- package/dist/version.js.map +1 -0
- package/dist/watcher.d.ts.map +1 -1
- package/dist/watcher.js +22 -1
- package/dist/watcher.js.map +1 -1
- package/dist/watcher.test.js +29 -0
- package/dist/watcher.test.js.map +1 -1
- package/package.json +1 -1
- package/src/bin/sync-runner.test.ts +131 -41
- package/src/bin/sync-runner.ts +56 -48
- package/src/cli/share.test.ts +169 -0
- package/src/cli/share.ts +81 -0
- package/src/cli/sync.ts +21 -88
- package/src/cli/tombstones.ts +106 -0
- package/src/context.test.ts +139 -1
- package/src/context.ts +59 -17
- package/src/entity-resolver.test.ts +3 -3
- package/src/index.ts +2 -0
- package/src/object-io.ts +12 -0
- package/src/personal-vault.test.ts +175 -0
- package/src/personal-vault.ts +86 -1
- package/src/signals/get.test.ts +83 -1
- package/src/signals/get.ts +9 -13
- package/src/signals/internals.ts +153 -4
- package/src/signals/list.test.ts +114 -1
- package/src/signals/list.ts +16 -29
- package/src/signals/types.ts +18 -1
- package/src/sources/get.test.ts +104 -1
- package/src/sources/get.ts +12 -24
- package/src/sources/internals.ts +156 -4
- package/src/sources/list.test.ts +132 -1
- package/src/sources/list.ts +16 -29
- package/src/sources/types.ts +18 -1
- package/src/sync/event-sync.ts +6 -7
- package/src/types.ts +33 -3
- package/src/version.ts +24 -0
- package/src/watcher.test.ts +41 -0
- package/src/watcher.ts +24 -1
package/src/sources/get.test.ts
CHANGED
|
@@ -2,13 +2,14 @@
|
|
|
2
2
|
* Unit tests for sources/get.ts.
|
|
3
3
|
*/
|
|
4
4
|
|
|
5
|
-
import { describe, it, expect, beforeEach, afterEach } from "vitest";
|
|
5
|
+
import { describe, it, expect, vi, beforeEach, afterEach } from "vitest";
|
|
6
6
|
import { GetObjectCommand, type S3Client } from "@aws-sdk/client-s3";
|
|
7
7
|
import { getSource, SourceNotFoundError } from "./get.js";
|
|
8
8
|
import {
|
|
9
9
|
_setSourcesS3Factory,
|
|
10
10
|
_resetSourcesS3Factory,
|
|
11
11
|
} from "./internals.js";
|
|
12
|
+
import type { PresignTransportClient } from "../object-io.js";
|
|
12
13
|
import type { EntityContext } from "../types.js";
|
|
13
14
|
import { InvalidSourceChannelError } from "../schemas/source-channels.js";
|
|
14
15
|
|
|
@@ -164,3 +165,105 @@ describe("getSource", () => {
|
|
|
164
165
|
expect(doc.body).toBe("Just a body, no frontmatter.");
|
|
165
166
|
});
|
|
166
167
|
});
|
|
168
|
+
|
|
169
|
+
// ---------------------------------------------------------------------------
|
|
170
|
+
// Presigned transport (vault client + company vault) — HQ-59
|
|
171
|
+
// ---------------------------------------------------------------------------
|
|
172
|
+
|
|
173
|
+
/**
|
|
174
|
+
* A presign-capable vault stub + a `fetch` mock keyed by the presigned URL.
|
|
175
|
+
* presign() mints a URL that encodes the key; fetch() returns the object's
|
|
176
|
+
* bytes (404 when absent, so a missing key surfaces as SourceNotFoundError —
|
|
177
|
+
* matching the S3 NoSuchKey path).
|
|
178
|
+
*/
|
|
179
|
+
function makePresignVault(objects: Record<string, string>): PresignTransportClient {
|
|
180
|
+
vi.stubGlobal(
|
|
181
|
+
"fetch",
|
|
182
|
+
vi.fn(async (url: string) => {
|
|
183
|
+
const key = new URL(url).searchParams.get("key") ?? "";
|
|
184
|
+
const content = objects[key];
|
|
185
|
+
if (content === undefined) return new Response(null, { status: 404 });
|
|
186
|
+
return new Response(Buffer.from(content, "utf-8"), { status: 200 });
|
|
187
|
+
}),
|
|
188
|
+
);
|
|
189
|
+
return {
|
|
190
|
+
presign: async (input) => ({
|
|
191
|
+
results: input.keys.map((k) => ({
|
|
192
|
+
key: k.key,
|
|
193
|
+
op: k.op ?? input.op ?? "get",
|
|
194
|
+
url: `https://signed.example/?key=${encodeURIComponent(k.key)}`,
|
|
195
|
+
})),
|
|
196
|
+
expiresAt: "2099-01-01T00:00:00.000Z",
|
|
197
|
+
}),
|
|
198
|
+
listFiles: async () => ({ objects: [], cursor: null, truncated: false }),
|
|
199
|
+
};
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
describe("getSource — presigned transport (vault + cmp_)", () => {
|
|
203
|
+
afterEach(() => vi.unstubAllGlobals());
|
|
204
|
+
|
|
205
|
+
it("reads via presign and never touches S3", async () => {
|
|
206
|
+
// S3 factory throws if used — proves the read went through presign.
|
|
207
|
+
_setSourcesS3Factory(
|
|
208
|
+
() =>
|
|
209
|
+
({
|
|
210
|
+
send: async () => {
|
|
211
|
+
throw new Error("S3 must not be used on the presign path");
|
|
212
|
+
},
|
|
213
|
+
}) as unknown as S3Client,
|
|
214
|
+
);
|
|
215
|
+
const vault = makePresignVault({ "sources/meetings/abc.md": MEETING_MD });
|
|
216
|
+
|
|
217
|
+
const doc = await getSource({
|
|
218
|
+
entity: ENTITY,
|
|
219
|
+
channel: "meeting",
|
|
220
|
+
sourceId: "abc",
|
|
221
|
+
vault,
|
|
222
|
+
});
|
|
223
|
+
|
|
224
|
+
expect(doc.frontmatter).toEqual({
|
|
225
|
+
source_id: "abc",
|
|
226
|
+
source_type: "meeting",
|
|
227
|
+
title: "Q1 Planning",
|
|
228
|
+
});
|
|
229
|
+
expect(doc.body).toContain("Body text line 1.");
|
|
230
|
+
});
|
|
231
|
+
|
|
232
|
+
it("maps a presign 404 to SourceNotFoundError", async () => {
|
|
233
|
+
const vault = makePresignVault({});
|
|
234
|
+
await expect(
|
|
235
|
+
getSource({ entity: ENTITY, channel: "meeting", sourceId: "missing", vault }),
|
|
236
|
+
).rejects.toBeInstanceOf(SourceNotFoundError);
|
|
237
|
+
});
|
|
238
|
+
|
|
239
|
+
it("includeRaw via presign reads the .raw.json sibling", async () => {
|
|
240
|
+
const vault = makePresignVault({
|
|
241
|
+
"sources/meetings/abc.md": MEETING_MD,
|
|
242
|
+
"sources/meetings/abc.raw.json": JSON.stringify(RAW_PAYLOAD),
|
|
243
|
+
});
|
|
244
|
+
const doc = await getSource({
|
|
245
|
+
entity: ENTITY,
|
|
246
|
+
channel: "meeting",
|
|
247
|
+
sourceId: "abc",
|
|
248
|
+
includeRaw: true,
|
|
249
|
+
vault,
|
|
250
|
+
});
|
|
251
|
+
expect(doc.raw).toEqual(RAW_PAYLOAD);
|
|
252
|
+
});
|
|
253
|
+
|
|
254
|
+
it("personal vault (prs_) ignores the vault client and stays on direct S3", async () => {
|
|
255
|
+
// S3 stub serves the doc; the presign fetch would 404 if (wrongly) used.
|
|
256
|
+
installStub({ "sources/meetings/abc.md": MEETING_MD });
|
|
257
|
+
const vault = makePresignVault({});
|
|
258
|
+
const personal: EntityContext = { ...ENTITY, uid: "prs_me" };
|
|
259
|
+
|
|
260
|
+
const doc = await getSource({
|
|
261
|
+
entity: personal,
|
|
262
|
+
channel: "meeting",
|
|
263
|
+
sourceId: "abc",
|
|
264
|
+
vault,
|
|
265
|
+
});
|
|
266
|
+
|
|
267
|
+
expect(doc.body).toContain("Body text line 1.");
|
|
268
|
+
});
|
|
269
|
+
});
|
package/src/sources/get.ts
CHANGED
|
@@ -3,9 +3,8 @@
|
|
|
3
3
|
* sibling) from an entity bucket and return parsed frontmatter + body.
|
|
4
4
|
*/
|
|
5
5
|
|
|
6
|
-
import { GetObjectCommand } from "@aws-sdk/client-s3";
|
|
7
6
|
import { assertSourceChannel, sourcePathSegment } from "../schemas/source-channels.js";
|
|
8
|
-
import {
|
|
7
|
+
import { readObjectText, tryReadObjectText } from "./internals.js";
|
|
9
8
|
import { parseMarkdown } from "./parse.js";
|
|
10
9
|
import type { GetSourceOptions, SourceDocument } from "./types.js";
|
|
11
10
|
|
|
@@ -19,15 +18,18 @@ export class SourceNotFoundError extends Error {
|
|
|
19
18
|
function isNoSuchKey(err: unknown): boolean {
|
|
20
19
|
if (!err || typeof err !== "object") return false;
|
|
21
20
|
const name = (err as { name?: unknown }).name;
|
|
22
|
-
//
|
|
23
|
-
|
|
21
|
+
// The transport normalizes a missing object to a `NoSuchKey`-named error on
|
|
22
|
+
// both the S3 and presigned paths (a presign 404 → NoSuchKey). `NotFound` is
|
|
23
|
+
// kept for defense in depth (S3 HEAD-style errors).
|
|
24
|
+
return (
|
|
25
|
+
name === "NoSuchKey" || name === "NoSuchKeyException" || name === "NotFound"
|
|
26
|
+
);
|
|
24
27
|
}
|
|
25
28
|
|
|
26
29
|
export async function getSource(opts: GetSourceOptions): Promise<SourceDocument> {
|
|
27
|
-
// Validate channel BEFORE any
|
|
30
|
+
// Validate channel BEFORE any read (acceptance criterion).
|
|
28
31
|
assertSourceChannel(opts.channel);
|
|
29
32
|
|
|
30
|
-
const client = getS3Client(opts.entity);
|
|
31
33
|
// sources-pipeline writes "meeting" channel to sources/meetings/...; other
|
|
32
34
|
// channels pass through. See schemas/source-channels.ts:sourcePathSegment.
|
|
33
35
|
const segment = sourcePathSegment(opts.channel);
|
|
@@ -35,13 +37,7 @@ export async function getSource(opts: GetSourceOptions): Promise<SourceDocument>
|
|
|
35
37
|
|
|
36
38
|
let body: string;
|
|
37
39
|
try {
|
|
38
|
-
|
|
39
|
-
new GetObjectCommand({
|
|
40
|
-
Bucket: opts.entity.bucketName,
|
|
41
|
-
Key: key,
|
|
42
|
-
}),
|
|
43
|
-
);
|
|
44
|
-
body = await streamToString(response.Body);
|
|
40
|
+
body = await readObjectText(opts, key);
|
|
45
41
|
} catch (err) {
|
|
46
42
|
if (isNoSuchKey(err)) {
|
|
47
43
|
throw new SourceNotFoundError(key);
|
|
@@ -58,19 +54,11 @@ export async function getSource(opts: GetSourceOptions): Promise<SourceDocument>
|
|
|
58
54
|
};
|
|
59
55
|
|
|
60
56
|
if (opts.includeRaw) {
|
|
57
|
+
// Raw sibling is optional; tryReadObjectText returns null when absent.
|
|
61
58
|
const rawKey = `sources/${segment}/${opts.sourceId}.raw.json`;
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
new GetObjectCommand({
|
|
65
|
-
Bucket: opts.entity.bucketName,
|
|
66
|
-
Key: rawKey,
|
|
67
|
-
}),
|
|
68
|
-
);
|
|
69
|
-
const rawText = await streamToString(rawResponse.Body);
|
|
59
|
+
const rawText = await tryReadObjectText(opts, rawKey);
|
|
60
|
+
if (rawText !== null) {
|
|
70
61
|
document.raw = JSON.parse(rawText);
|
|
71
|
-
} catch (err) {
|
|
72
|
-
if (!isNoSuchKey(err)) throw err;
|
|
73
|
-
// Raw sibling is optional; leave undefined when absent.
|
|
74
62
|
}
|
|
75
63
|
}
|
|
76
64
|
|
package/src/sources/internals.ts
CHANGED
|
@@ -1,15 +1,37 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Internal helpers shared between sources/list.ts and sources/get.ts.
|
|
3
3
|
*
|
|
4
|
-
*
|
|
5
|
-
*
|
|
6
|
-
*
|
|
4
|
+
* Two transports back the read surface:
|
|
5
|
+
* - presigned-URL (HQ-59): when a vault client is supplied AND the entity is a
|
|
6
|
+
* company vault (cmp_*), reads go through the vault `list`/`presign`
|
|
7
|
+
* endpoints — the client never talks to S3 directly. This is the path every
|
|
8
|
+
* company read should take going forward.
|
|
9
|
+
* - direct S3 (legacy / personal): no vault client, or a personal vault
|
|
10
|
+
* (prs_*, whose membership-less creds 403 the membership-gated presign
|
|
11
|
+
* endpoints). Goes through `getS3Client`, whose factory hook lets tests
|
|
12
|
+
* inject an in-memory stub without vi.mock.
|
|
7
13
|
*/
|
|
8
14
|
|
|
9
|
-
import {
|
|
15
|
+
import {
|
|
16
|
+
S3Client,
|
|
17
|
+
GetObjectCommand,
|
|
18
|
+
ListObjectsV2Command,
|
|
19
|
+
} from "@aws-sdk/client-s3";
|
|
10
20
|
import type { EntityContext } from "../types.js";
|
|
21
|
+
import { PresignObjectIO, type PresignTransportClient } from "../object-io.js";
|
|
11
22
|
|
|
12
23
|
function defaultFactory(ctx: EntityContext): S3Client {
|
|
24
|
+
if (!ctx.credentials) {
|
|
25
|
+
// The direct-S3 read path needs STS creds. A credential-less context only
|
|
26
|
+
// exists on the presign path (HQ-59 company vend-skip), which reads via the
|
|
27
|
+
// vault list/presign endpoints, not this S3 client — so arriving here with
|
|
28
|
+
// no creds is a routing bug. Fail loudly rather than 403 opaquely later.
|
|
29
|
+
throw new Error(
|
|
30
|
+
`Sources S3 read for ${ctx.uid} requires STS credentials but got a ` +
|
|
31
|
+
`presign-only context; company presign reads must go through the ` +
|
|
32
|
+
`vault transport, not getS3Client.`,
|
|
33
|
+
);
|
|
34
|
+
}
|
|
13
35
|
return new S3Client({
|
|
14
36
|
region: ctx.region,
|
|
15
37
|
credentials: {
|
|
@@ -44,3 +66,133 @@ export async function streamToString(body: unknown): Promise<string> {
|
|
|
44
66
|
}
|
|
45
67
|
return Buffer.concat(chunks).toString("utf-8");
|
|
46
68
|
}
|
|
69
|
+
|
|
70
|
+
// ---------------------------------------------------------------------------
|
|
71
|
+
// Transport seam (presign for cmp_, direct-S3 otherwise)
|
|
72
|
+
// ---------------------------------------------------------------------------
|
|
73
|
+
|
|
74
|
+
/** Common shape of every reader's options: an entity + an optional vault client. */
|
|
75
|
+
export interface ReaderTransportOpts {
|
|
76
|
+
entity: EntityContext;
|
|
77
|
+
/**
|
|
78
|
+
* Presign-capable vault client. When supplied and the entity is a company
|
|
79
|
+
* vault (cmp_*), reads use the presigned-URL transport instead of direct S3.
|
|
80
|
+
*/
|
|
81
|
+
vault?: PresignTransportClient;
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
/** Lightweight listed object (transport-agnostic). */
|
|
85
|
+
export interface ListedKey {
|
|
86
|
+
key: string;
|
|
87
|
+
size: number;
|
|
88
|
+
lastModified: Date;
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
/** True when this read should use the presigned-URL transport. */
|
|
92
|
+
export function usePresign(opts: ReaderTransportOpts): boolean {
|
|
93
|
+
return opts.vault != null && opts.entity.uid.startsWith("cmp_");
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
/** A not-found error normalized to the S3 `NoSuchKey` shape callers test for. */
|
|
97
|
+
function noSuchKeyError(key: string): Error {
|
|
98
|
+
return Object.assign(new Error(`No such key: ${key}`), { name: "NoSuchKey" });
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
function isNotFound(err: unknown): boolean {
|
|
102
|
+
if (!err || typeof err !== "object") return false;
|
|
103
|
+
const name = (err as { name?: unknown }).name;
|
|
104
|
+
return (
|
|
105
|
+
name === "NoSuchKey" || name === "NoSuchKeyException" || name === "NotFound"
|
|
106
|
+
);
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
/**
|
|
110
|
+
* Read an object's UTF-8 text. Throws a `NoSuchKey`-named error when the object
|
|
111
|
+
* is absent (both transports), so callers' existing not-found checks work
|
|
112
|
+
* unchanged.
|
|
113
|
+
*/
|
|
114
|
+
export async function readObjectText(
|
|
115
|
+
opts: ReaderTransportOpts,
|
|
116
|
+
key: string,
|
|
117
|
+
): Promise<string> {
|
|
118
|
+
if (usePresign(opts)) {
|
|
119
|
+
const io = new PresignObjectIO(opts.vault!, opts.entity.uid);
|
|
120
|
+
try {
|
|
121
|
+
const { body } = await io.getObject(key);
|
|
122
|
+
return body.toString("utf-8");
|
|
123
|
+
} catch (err) {
|
|
124
|
+
if (isNotFound(err)) throw noSuchKeyError(key);
|
|
125
|
+
throw err;
|
|
126
|
+
}
|
|
127
|
+
}
|
|
128
|
+
const client = getS3Client(opts.entity);
|
|
129
|
+
const res = await client.send(
|
|
130
|
+
new GetObjectCommand({ Bucket: opts.entity.bucketName, Key: key }),
|
|
131
|
+
);
|
|
132
|
+
return streamToString(res.Body);
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
/** Like {@link readObjectText} but returns null when the object is absent. */
|
|
136
|
+
export async function tryReadObjectText(
|
|
137
|
+
opts: ReaderTransportOpts,
|
|
138
|
+
key: string,
|
|
139
|
+
): Promise<string | null> {
|
|
140
|
+
try {
|
|
141
|
+
return await readObjectText(opts, key);
|
|
142
|
+
} catch (err) {
|
|
143
|
+
if (isNotFound(err)) return null;
|
|
144
|
+
throw err;
|
|
145
|
+
}
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
/**
|
|
149
|
+
* List one page of objects under `prefix`.
|
|
150
|
+
*
|
|
151
|
+
* Under the presigned transport the vault `list` endpoint is ACL-filtered and
|
|
152
|
+
* controls page size server-side, so `limit` is advisory — paginate via the
|
|
153
|
+
* returned `nextToken` (the opaque vault cursor) until it is undefined. Under
|
|
154
|
+
* direct S3, `limit` maps to `MaxKeys`.
|
|
155
|
+
*/
|
|
156
|
+
export async function listObjects(
|
|
157
|
+
opts: ReaderTransportOpts,
|
|
158
|
+
prefix: string,
|
|
159
|
+
page: { limit?: number; continuationToken?: string },
|
|
160
|
+
): Promise<{ contents: ListedKey[]; nextToken?: string }> {
|
|
161
|
+
if (usePresign(opts)) {
|
|
162
|
+
const io = new PresignObjectIO(opts.vault!, opts.entity.uid);
|
|
163
|
+
const res = await io.listObjects({
|
|
164
|
+
prefix,
|
|
165
|
+
continuationToken: page.continuationToken,
|
|
166
|
+
});
|
|
167
|
+
return {
|
|
168
|
+
contents: res.objects.map((o) => ({
|
|
169
|
+
key: o.key,
|
|
170
|
+
size: o.size,
|
|
171
|
+
lastModified: o.lastModified,
|
|
172
|
+
})),
|
|
173
|
+
nextToken: res.nextContinuationToken,
|
|
174
|
+
};
|
|
175
|
+
}
|
|
176
|
+
const client = getS3Client(opts.entity);
|
|
177
|
+
const res = await client.send(
|
|
178
|
+
new ListObjectsV2Command({
|
|
179
|
+
Bucket: opts.entity.bucketName,
|
|
180
|
+
Prefix: prefix,
|
|
181
|
+
MaxKeys: page.limit,
|
|
182
|
+
ContinuationToken: page.continuationToken,
|
|
183
|
+
}),
|
|
184
|
+
);
|
|
185
|
+
const contents: ListedKey[] = [];
|
|
186
|
+
for (const obj of res.Contents ?? []) {
|
|
187
|
+
if (!obj.Key) continue;
|
|
188
|
+
contents.push({
|
|
189
|
+
key: obj.Key,
|
|
190
|
+
size: obj.Size ?? 0,
|
|
191
|
+
lastModified: obj.LastModified ?? new Date(0),
|
|
192
|
+
});
|
|
193
|
+
}
|
|
194
|
+
return {
|
|
195
|
+
contents,
|
|
196
|
+
nextToken: res.IsTruncated ? res.NextContinuationToken : undefined,
|
|
197
|
+
};
|
|
198
|
+
}
|
package/src/sources/list.test.ts
CHANGED
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
* hook so we don't have to stand up vi.mock for @aws-sdk/client-s3.
|
|
6
6
|
*/
|
|
7
7
|
|
|
8
|
-
import { describe, it, expect, beforeEach, afterEach } from "vitest";
|
|
8
|
+
import { describe, it, expect, vi, beforeEach, afterEach } from "vitest";
|
|
9
9
|
import {
|
|
10
10
|
GetObjectCommand,
|
|
11
11
|
ListObjectsV2Command,
|
|
@@ -16,6 +16,8 @@ import {
|
|
|
16
16
|
_setSourcesS3Factory,
|
|
17
17
|
_resetSourcesS3Factory,
|
|
18
18
|
} from "./internals.js";
|
|
19
|
+
import type { PresignTransportClient } from "../object-io.js";
|
|
20
|
+
import type { VaultListedObject } from "../vault-client.js";
|
|
19
21
|
import type { EntityContext } from "../types.js";
|
|
20
22
|
import { InvalidSourceChannelError } from "../schemas/source-channels.js";
|
|
21
23
|
|
|
@@ -245,3 +247,132 @@ describe("listSources", () => {
|
|
|
245
247
|
expect(observed.commands).toHaveLength(0);
|
|
246
248
|
});
|
|
247
249
|
});
|
|
250
|
+
|
|
251
|
+
// ---------------------------------------------------------------------------
|
|
252
|
+
// Presigned transport (vault client + company vault) — HQ-59
|
|
253
|
+
// ---------------------------------------------------------------------------
|
|
254
|
+
|
|
255
|
+
interface PresignVaultStub {
|
|
256
|
+
vault: PresignTransportClient;
|
|
257
|
+
listCalls: Array<{ prefix?: string; cursor?: string }>;
|
|
258
|
+
}
|
|
259
|
+
|
|
260
|
+
/**
|
|
261
|
+
* Vault stub for the list path: `listFiles` returns the configured ACL-filtered
|
|
262
|
+
* page (the server is the ACL boundary — only readable keys come back) + an
|
|
263
|
+
* opaque cursor; `presign`+fetch serve per-entry frontmatter GETs.
|
|
264
|
+
*/
|
|
265
|
+
function makePresignVault(
|
|
266
|
+
page: { objects: VaultListedObject[]; cursor: string | null },
|
|
267
|
+
contents: Record<string, string> = {},
|
|
268
|
+
): PresignVaultStub {
|
|
269
|
+
const listCalls: Array<{ prefix?: string; cursor?: string }> = [];
|
|
270
|
+
vi.stubGlobal(
|
|
271
|
+
"fetch",
|
|
272
|
+
vi.fn(async (url: string) => {
|
|
273
|
+
const key = new URL(url).searchParams.get("key") ?? "";
|
|
274
|
+
const c = contents[key];
|
|
275
|
+
if (c === undefined) return new Response(null, { status: 404 });
|
|
276
|
+
return new Response(Buffer.from(c, "utf-8"), { status: 200 });
|
|
277
|
+
}),
|
|
278
|
+
);
|
|
279
|
+
const vault: PresignTransportClient = {
|
|
280
|
+
presign: async (input) => ({
|
|
281
|
+
results: input.keys.map((k) => ({
|
|
282
|
+
key: k.key,
|
|
283
|
+
op: k.op ?? input.op ?? "get",
|
|
284
|
+
url: `https://signed.example/?key=${encodeURIComponent(k.key)}`,
|
|
285
|
+
})),
|
|
286
|
+
expiresAt: "2099-01-01T00:00:00.000Z",
|
|
287
|
+
}),
|
|
288
|
+
listFiles: async (_company, prefix, cursor) => {
|
|
289
|
+
listCalls.push({ prefix, cursor });
|
|
290
|
+
return {
|
|
291
|
+
objects: page.objects,
|
|
292
|
+
cursor: page.cursor,
|
|
293
|
+
truncated: page.cursor != null,
|
|
294
|
+
};
|
|
295
|
+
},
|
|
296
|
+
};
|
|
297
|
+
return { vault, listCalls };
|
|
298
|
+
}
|
|
299
|
+
|
|
300
|
+
function listed(key: string): VaultListedObject {
|
|
301
|
+
return {
|
|
302
|
+
key,
|
|
303
|
+
size: 10,
|
|
304
|
+
lastModified: "2026-02-01T00:00:00.000Z",
|
|
305
|
+
etag: "etag",
|
|
306
|
+
permission: "read",
|
|
307
|
+
};
|
|
308
|
+
}
|
|
309
|
+
|
|
310
|
+
describe("listSources — presigned transport (vault + cmp_)", () => {
|
|
311
|
+
afterEach(() => vi.unstubAllGlobals());
|
|
312
|
+
|
|
313
|
+
it("lists via the ACL-filtered vault endpoint and never touches S3", async () => {
|
|
314
|
+
_setSourcesS3Factory(
|
|
315
|
+
() =>
|
|
316
|
+
({
|
|
317
|
+
send: async () => {
|
|
318
|
+
throw new Error("S3 must not be used on the presign path");
|
|
319
|
+
},
|
|
320
|
+
}) as unknown as S3Client,
|
|
321
|
+
);
|
|
322
|
+
const { vault, listCalls } = makePresignVault({
|
|
323
|
+
objects: [listed("sources/meetings/a.md"), listed("sources/meetings/a.raw.json")],
|
|
324
|
+
cursor: null,
|
|
325
|
+
});
|
|
326
|
+
|
|
327
|
+
const res = await listSources({ entity: ENTITY, channel: "meeting", vault });
|
|
328
|
+
|
|
329
|
+
// .raw.json sibling is skipped; only the .md entry surfaces.
|
|
330
|
+
expect(res.entries.map((e) => e.sourceId)).toEqual(["a"]);
|
|
331
|
+
expect(res.entries[0].key).toBe("sources/meetings/a.md");
|
|
332
|
+
expect(res.nextToken).toBeUndefined();
|
|
333
|
+
// listFiles was called with the channel prefix (ACL filtering is server-side).
|
|
334
|
+
expect(listCalls[0].prefix).toBe("sources/meetings/");
|
|
335
|
+
});
|
|
336
|
+
|
|
337
|
+
it("surfaces the opaque vault cursor as nextToken (advisory limit)", async () => {
|
|
338
|
+
const { vault } = makePresignVault({
|
|
339
|
+
objects: [listed("sources/meetings/a.md")],
|
|
340
|
+
cursor: "OPAQUE_CURSOR",
|
|
341
|
+
});
|
|
342
|
+
const res = await listSources({
|
|
343
|
+
entity: ENTITY,
|
|
344
|
+
channel: "meeting",
|
|
345
|
+
limit: 1,
|
|
346
|
+
vault,
|
|
347
|
+
});
|
|
348
|
+
expect(res.nextToken).toBe("OPAQUE_CURSOR");
|
|
349
|
+
});
|
|
350
|
+
|
|
351
|
+
it("includeFrontmatter parses each entry via a presigned GET", async () => {
|
|
352
|
+
const { vault } = makePresignVault(
|
|
353
|
+
{ objects: [listed("sources/meetings/a.md")], cursor: null },
|
|
354
|
+
{ "sources/meetings/a.md": MEETING_MD },
|
|
355
|
+
);
|
|
356
|
+
const res = await listSources({
|
|
357
|
+
entity: ENTITY,
|
|
358
|
+
channel: "meeting",
|
|
359
|
+
includeFrontmatter: true,
|
|
360
|
+
vault,
|
|
361
|
+
});
|
|
362
|
+
expect(res.entries[0].frontmatter).toMatchObject({ title: "Hello" });
|
|
363
|
+
});
|
|
364
|
+
|
|
365
|
+
it("forwards the continuationToken as the vault cursor", async () => {
|
|
366
|
+
const { vault, listCalls } = makePresignVault({
|
|
367
|
+
objects: [],
|
|
368
|
+
cursor: null,
|
|
369
|
+
});
|
|
370
|
+
await listSources({
|
|
371
|
+
entity: ENTITY,
|
|
372
|
+
channel: "meeting",
|
|
373
|
+
continuationToken: "RESUME_HERE",
|
|
374
|
+
vault,
|
|
375
|
+
});
|
|
376
|
+
expect(listCalls[0].cursor).toBe("RESUME_HERE");
|
|
377
|
+
});
|
|
378
|
+
});
|
package/src/sources/list.ts
CHANGED
|
@@ -6,12 +6,8 @@
|
|
|
6
6
|
* the summary (callers fetch them via getSource({ includeRaw: true })).
|
|
7
7
|
*/
|
|
8
8
|
|
|
9
|
-
import {
|
|
10
|
-
GetObjectCommand,
|
|
11
|
-
ListObjectsV2Command,
|
|
12
|
-
} from "@aws-sdk/client-s3";
|
|
13
9
|
import { assertSourceChannel, sourcePathSegment } from "../schemas/source-channels.js";
|
|
14
|
-
import {
|
|
10
|
+
import { listObjects, readObjectText } from "./internals.js";
|
|
15
11
|
import { parseFrontmatter } from "./parse.js";
|
|
16
12
|
import type {
|
|
17
13
|
ListSourcesOptions,
|
|
@@ -40,46 +36,37 @@ function deriveSourceId(
|
|
|
40
36
|
}
|
|
41
37
|
|
|
42
38
|
export async function listSources(opts: ListSourcesOptions): Promise<ListSourcesResult> {
|
|
43
|
-
// Validate channel BEFORE any
|
|
39
|
+
// Validate channel BEFORE any read (acceptance criterion).
|
|
44
40
|
assertSourceChannel(opts.channel);
|
|
45
41
|
|
|
46
|
-
const client = getS3Client(opts.entity);
|
|
47
42
|
// Use the canonical path segment from the schema; channel "meeting" maps
|
|
48
43
|
// to the plural "meetings/" prefix the sources-pipeline writes to.
|
|
49
44
|
const prefix = `sources/${sourcePathSegment(opts.channel)}/`;
|
|
50
45
|
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
);
|
|
46
|
+
// Presign transport ACL-filters server-side and controls page size, so
|
|
47
|
+
// `limit` is advisory there; under direct S3 it maps to MaxKeys. Paginate via
|
|
48
|
+
// the returned nextToken regardless of transport.
|
|
49
|
+
const page = await listObjects(opts, prefix, {
|
|
50
|
+
limit: opts.limit,
|
|
51
|
+
continuationToken: opts.continuationToken,
|
|
52
|
+
});
|
|
59
53
|
|
|
60
54
|
const entries: SourceSummary[] = [];
|
|
61
|
-
for (const obj of
|
|
62
|
-
|
|
63
|
-
const derived = deriveSourceId(obj.Key, prefix);
|
|
55
|
+
for (const obj of page.contents) {
|
|
56
|
+
const derived = deriveSourceId(obj.key, prefix);
|
|
64
57
|
if (!derived || derived.isRawSibling) continue;
|
|
65
58
|
|
|
66
59
|
const summary: SourceSummary = {
|
|
67
60
|
sourceId: derived.sourceId,
|
|
68
61
|
channel: opts.channel,
|
|
69
|
-
key: obj.
|
|
70
|
-
lastModified: obj.
|
|
71
|
-
size: obj.
|
|
62
|
+
key: obj.key,
|
|
63
|
+
lastModified: obj.lastModified,
|
|
64
|
+
size: obj.size,
|
|
72
65
|
};
|
|
73
66
|
|
|
74
67
|
if (opts.includeFrontmatter) {
|
|
75
68
|
try {
|
|
76
|
-
const
|
|
77
|
-
new GetObjectCommand({
|
|
78
|
-
Bucket: opts.entity.bucketName,
|
|
79
|
-
Key: obj.Key,
|
|
80
|
-
}),
|
|
81
|
-
);
|
|
82
|
-
const text = await streamToString(get.Body);
|
|
69
|
+
const text = await readObjectText(opts, obj.key);
|
|
83
70
|
const fm = parseFrontmatter(text);
|
|
84
71
|
if (fm) summary.frontmatter = fm;
|
|
85
72
|
} catch {
|
|
@@ -92,6 +79,6 @@ export async function listSources(opts: ListSourcesOptions): Promise<ListSources
|
|
|
92
79
|
|
|
93
80
|
return {
|
|
94
81
|
entries,
|
|
95
|
-
nextToken:
|
|
82
|
+
nextToken: page.nextToken,
|
|
96
83
|
};
|
|
97
84
|
}
|
package/src/sources/types.ts
CHANGED
|
@@ -6,6 +6,7 @@
|
|
|
6
6
|
*/
|
|
7
7
|
|
|
8
8
|
import type { SourceChannel } from "../schemas/source-channels.js";
|
|
9
|
+
import type { PresignTransportClient } from "../object-io.js";
|
|
9
10
|
|
|
10
11
|
/**
|
|
11
12
|
* Lightweight summary returned by listSources(). `frontmatter` is only
|
|
@@ -44,12 +45,23 @@ export interface SourceDocument {
|
|
|
44
45
|
export interface ListSourcesOptions {
|
|
45
46
|
entity: import("../types.js").EntityContext;
|
|
46
47
|
channel: SourceChannel;
|
|
47
|
-
/**
|
|
48
|
+
/**
|
|
49
|
+
* Max keys per page. Under direct S3 this maps to `MaxKeys`; under the
|
|
50
|
+
* presigned transport ({@link vault} + a company vault) it is advisory — the
|
|
51
|
+
* vault `list` endpoint controls page size server-side. Defaults to the S3
|
|
52
|
+
* default (1000) on the direct path.
|
|
53
|
+
*/
|
|
48
54
|
limit?: number;
|
|
49
55
|
/** Opaque continuation token returned by a prior page. */
|
|
50
56
|
continuationToken?: string;
|
|
51
57
|
/** When true, fetches+parses each entry's frontmatter (extra GETs). */
|
|
52
58
|
includeFrontmatter?: boolean;
|
|
59
|
+
/**
|
|
60
|
+
* Presign-capable vault client. When supplied and {@link entity} is a company
|
|
61
|
+
* vault (cmp_*), reads use the presigned-URL transport (ACL-filtered, no
|
|
62
|
+
* direct S3). Personal vaults and the absent-client path stay on direct S3.
|
|
63
|
+
*/
|
|
64
|
+
vault?: PresignTransportClient;
|
|
53
65
|
}
|
|
54
66
|
|
|
55
67
|
export interface ListSourcesResult {
|
|
@@ -64,4 +76,9 @@ export interface GetSourceOptions {
|
|
|
64
76
|
sourceId: string;
|
|
65
77
|
/** When true, also fetches the `.raw.json` sibling. */
|
|
66
78
|
includeRaw?: boolean;
|
|
79
|
+
/**
|
|
80
|
+
* Presign-capable vault client. When supplied and {@link entity} is a company
|
|
81
|
+
* vault (cmp_*), reads use the presigned-URL transport (no direct S3).
|
|
82
|
+
*/
|
|
83
|
+
vault?: PresignTransportClient;
|
|
67
84
|
}
|
package/src/sync/event-sync.ts
CHANGED
|
@@ -7,9 +7,8 @@
|
|
|
7
7
|
* runner's receiver seam defaults to {@link NoopPushReceiver}. This module is
|
|
8
8
|
* the connective tissue that turns both on for enrolled accounts:
|
|
9
9
|
*
|
|
10
|
-
* - {@link resolveEventSync} — the rollout gate. EXACT-email allowlist
|
|
11
|
-
*
|
|
12
|
-
* address, not domain suffix) + `HQ_SYNC_EVENT_SYNC` env override in both
|
|
10
|
+
* - {@link resolveEventSync} — the rollout gate. EXACT-email allowlist (full
|
|
11
|
+
* address, not a domain suffix) + `HQ_SYNC_EVENT_SYNC` env override in both
|
|
13
12
|
* directions. ONE gate governs publish AND receive so a device is never
|
|
14
13
|
* publish-only or receive-only in a half-rolled state.
|
|
15
14
|
* - {@link subscribeSyncReceive} — `POST /v1/sync/subscribe` (US-015/US-016):
|
|
@@ -58,8 +57,8 @@ import type { TreeChangeBatch } from "../watcher.js";
|
|
|
58
57
|
* single-account Phase 3 rollout (2026-06-10) targets the operator's own
|
|
59
58
|
* devices; `xhassaan@getindigo.ai` and `hassaan@getindigo.ai.evil.com` must
|
|
60
59
|
* never match. Broadening later is an entry here, then a domain set, then a
|
|
61
|
-
* GA default-on switch (the path the presigned-URL transport
|
|
62
|
-
*
|
|
60
|
+
* GA default-on switch (the same rollout path the presigned-URL sync transport
|
|
61
|
+
* already took to GA).
|
|
63
62
|
*/
|
|
64
63
|
export const EVENT_SYNC_ROLLOUT_EMAILS: ReadonlySet<string> = new Set([
|
|
65
64
|
"hassaan@getindigo.ai",
|
|
@@ -68,8 +67,8 @@ export const EVENT_SYNC_ROLLOUT_EMAILS: ReadonlySet<string> = new Set([
|
|
|
68
67
|
/**
|
|
69
68
|
* Decide whether this session runs event-driven sync (publish + receive).
|
|
70
69
|
*
|
|
71
|
-
*
|
|
72
|
-
* overrides
|
|
70
|
+
* Env-override-in-both-directions precedence: `HQ_SYNC_EVENT_SYNC`
|
|
71
|
+
* overrides (`1`/`true`/`yes`/`on` → force on,
|
|
73
72
|
* `0`/`false`/`no`/`off` → force off) so unenrolled testers can exercise it
|
|
74
73
|
* and enrolled accounts can be rolled back without a release. An unset/blank
|
|
75
74
|
* override falls through to the exact-email check; an unrecognized override
|