@indigoai-us/hq-cloud 6.11.10 → 6.11.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin/sync-runner.d.ts +2 -0
- package/dist/bin/sync-runner.d.ts.map +1 -1
- package/dist/bin/sync-runner.js +231 -52
- package/dist/bin/sync-runner.js.map +1 -1
- package/dist/bin/sync-runner.test.js +330 -11
- package/dist/bin/sync-runner.test.js.map +1 -1
- package/dist/cli/reindex.d.ts.map +1 -1
- package/dist/cli/reindex.js +16 -1
- package/dist/cli/reindex.js.map +1 -1
- package/dist/cli/reindex.test.js +39 -1
- package/dist/cli/reindex.test.js.map +1 -1
- package/dist/cli/rescue-classify-ordering.test.js +58 -0
- package/dist/cli/rescue-classify-ordering.test.js.map +1 -1
- package/dist/cli/rescue-core.js +229 -15
- package/dist/cli/rescue-core.js.map +1 -1
- package/dist/cli/rescue-exec-bit-preserve.test.d.ts +2 -0
- package/dist/cli/rescue-exec-bit-preserve.test.d.ts.map +1 -0
- package/dist/cli/rescue-exec-bit-preserve.test.js +169 -0
- package/dist/cli/rescue-exec-bit-preserve.test.js.map +1 -0
- package/dist/cli/share.d.ts +2 -1
- package/dist/cli/share.d.ts.map +1 -1
- package/dist/cli/share.js +100 -32
- package/dist/cli/share.js.map +1 -1
- package/dist/cli/share.test.js +30 -0
- package/dist/cli/share.test.js.map +1 -1
- package/dist/cli/sync.d.ts +28 -1
- package/dist/cli/sync.d.ts.map +1 -1
- package/dist/cli/sync.js +188 -59
- package/dist/cli/sync.js.map +1 -1
- package/dist/cli/sync.test.js +487 -1
- package/dist/cli/sync.test.js.map +1 -1
- package/dist/cognito-auth.d.ts.map +1 -1
- package/dist/cognito-auth.js +55 -10
- package/dist/cognito-auth.js.map +1 -1
- package/dist/cognito-auth.test.js +61 -0
- package/dist/cognito-auth.test.js.map +1 -1
- package/dist/index.d.ts +2 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -1
- package/dist/index.js.map +1 -1
- package/dist/journal.d.ts.map +1 -1
- package/dist/journal.js +93 -6
- package/dist/journal.js.map +1 -1
- package/dist/journal.test.js +59 -0
- package/dist/journal.test.js.map +1 -1
- package/dist/machine-auth.test.js +60 -2
- package/dist/machine-auth.test.js.map +1 -1
- package/dist/object-io.d.ts +37 -1
- package/dist/object-io.d.ts.map +1 -1
- package/dist/object-io.js +148 -29
- package/dist/object-io.js.map +1 -1
- package/dist/object-io.test.js +121 -0
- package/dist/object-io.test.js.map +1 -1
- package/dist/operation-lock.d.ts +8 -8
- package/dist/operation-lock.d.ts.map +1 -1
- package/dist/operation-lock.js +99 -32
- package/dist/operation-lock.js.map +1 -1
- package/dist/operation-lock.test.js +51 -4
- package/dist/operation-lock.test.js.map +1 -1
- package/dist/personal-vault.d.ts +8 -0
- package/dist/personal-vault.d.ts.map +1 -1
- package/dist/personal-vault.js +17 -3
- package/dist/personal-vault.js.map +1 -1
- package/dist/personal-vault.test.js +34 -0
- package/dist/personal-vault.test.js.map +1 -1
- package/dist/prefix-coalesce.d.ts +20 -9
- package/dist/prefix-coalesce.d.ts.map +1 -1
- package/dist/prefix-coalesce.js +124 -28
- package/dist/prefix-coalesce.js.map +1 -1
- package/dist/prefix-coalesce.test.js +57 -2
- package/dist/prefix-coalesce.test.js.map +1 -1
- package/dist/remote-pull.d.ts +6 -1
- package/dist/remote-pull.d.ts.map +1 -1
- package/dist/remote-pull.js +62 -13
- package/dist/remote-pull.js.map +1 -1
- package/dist/remote-pull.test.js +189 -0
- package/dist/remote-pull.test.js.map +1 -1
- package/dist/s3.d.ts +2 -0
- package/dist/s3.d.ts.map +1 -1
- package/dist/s3.js +197 -116
- package/dist/s3.js.map +1 -1
- package/dist/s3.test.js +109 -0
- package/dist/s3.test.js.map +1 -1
- package/dist/scope-shrink.d.ts +3 -2
- package/dist/scope-shrink.d.ts.map +1 -1
- package/dist/scope-shrink.js +1 -1
- package/dist/scope-shrink.js.map +1 -1
- package/dist/skill-telemetry.d.ts +1 -1
- package/dist/skill-telemetry.d.ts.map +1 -1
- package/dist/skill-telemetry.js +69 -9
- package/dist/skill-telemetry.js.map +1 -1
- package/dist/skill-telemetry.test.js +86 -0
- package/dist/skill-telemetry.test.js.map +1 -1
- package/dist/sync/event-sync.d.ts +6 -0
- package/dist/sync/event-sync.d.ts.map +1 -1
- package/dist/sync/event-sync.js +34 -1
- package/dist/sync/event-sync.js.map +1 -1
- package/dist/sync/event-sync.test.js +73 -0
- package/dist/sync/event-sync.test.js.map +1 -1
- package/dist/sync/metrics.d.ts +17 -1
- package/dist/sync/metrics.d.ts.map +1 -1
- package/dist/sync/metrics.js +32 -1
- package/dist/sync/metrics.js.map +1 -1
- package/dist/sync/metrics.test.js +74 -1
- package/dist/sync/metrics.test.js.map +1 -1
- package/dist/sync/pull-scope.d.ts.map +1 -1
- package/dist/sync/pull-scope.js +15 -7
- package/dist/sync/pull-scope.js.map +1 -1
- package/dist/sync/push-receiver.d.ts +6 -5
- package/dist/sync/push-receiver.d.ts.map +1 -1
- package/dist/sync/push-receiver.js +13 -15
- package/dist/sync/push-receiver.js.map +1 -1
- package/dist/sync/push-receiver.test.js +36 -1
- package/dist/sync/push-receiver.test.js.map +1 -1
- package/dist/telemetry.d.ts +1 -1
- package/dist/telemetry.d.ts.map +1 -1
- package/dist/telemetry.js +59 -6
- package/dist/telemetry.js.map +1 -1
- package/dist/telemetry.test.js +74 -0
- package/dist/telemetry.test.js.map +1 -1
- package/dist/types.d.ts +8 -0
- package/dist/types.d.ts.map +1 -1
- package/dist/watcher.d.ts +36 -0
- package/dist/watcher.d.ts.map +1 -1
- package/dist/watcher.js +152 -30
- package/dist/watcher.js.map +1 -1
- package/dist/watcher.test.js +103 -0
- package/dist/watcher.test.js.map +1 -1
- package/package.json +1 -1
- package/src/bin/sync-runner.test.ts +396 -11
- package/src/bin/sync-runner.ts +254 -52
- package/src/cli/reindex.test.ts +47 -1
- package/src/cli/reindex.ts +17 -1
- package/src/cli/rescue-classify-ordering.test.ts +61 -0
- package/src/cli/rescue-core.ts +261 -15
- package/src/cli/rescue-exec-bit-preserve.test.ts +187 -0
- package/src/cli/share.test.ts +38 -0
- package/src/cli/share.ts +103 -34
- package/src/cli/sync.test.ts +594 -1
- package/src/cli/sync.ts +229 -65
- package/src/cognito-auth.test.ts +77 -0
- package/src/cognito-auth.ts +73 -11
- package/src/index.ts +8 -0
- package/src/journal.test.ts +72 -0
- package/src/journal.ts +95 -8
- package/src/machine-auth.test.ts +64 -2
- package/src/object-io.test.ts +142 -0
- package/src/object-io.ts +182 -30
- package/src/operation-lock.test.ts +63 -4
- package/src/operation-lock.ts +99 -31
- package/src/personal-vault.test.ts +42 -0
- package/src/personal-vault.ts +18 -3
- package/src/prefix-coalesce.test.ts +71 -1
- package/src/prefix-coalesce.ts +155 -30
- package/src/remote-pull.test.ts +205 -0
- package/src/remote-pull.ts +77 -14
- package/src/s3.test.ts +126 -0
- package/src/s3.ts +237 -122
- package/src/scope-shrink.ts +6 -3
- package/src/skill-telemetry.test.ts +109 -0
- package/src/skill-telemetry.ts +82 -14
- package/src/sync/event-sync.test.ts +75 -0
- package/src/sync/event-sync.ts +54 -1
- package/src/sync/metrics.test.ts +81 -0
- package/src/sync/metrics.ts +59 -4
- package/src/sync/pull-scope.ts +23 -7
- package/src/sync/push-receiver.test.ts +38 -1
- package/src/sync/push-receiver.ts +15 -18
- package/src/telemetry.test.ts +85 -0
- package/src/telemetry.ts +69 -6
- package/src/types.ts +8 -0
- package/src/watcher.test.ts +117 -0
- package/src/watcher.ts +209 -33
package/src/s3.ts
CHANGED
|
@@ -8,8 +8,16 @@
|
|
|
8
8
|
|
|
9
9
|
import * as fs from "fs";
|
|
10
10
|
import * as path from "path";
|
|
11
|
+
import * as crypto from "crypto";
|
|
12
|
+
import { Readable, Transform } from "stream";
|
|
13
|
+
import { pipeline } from "stream/promises";
|
|
11
14
|
import type { EntityContext } from "./types.js";
|
|
12
|
-
import {
|
|
15
|
+
import {
|
|
16
|
+
resolveObjectIO,
|
|
17
|
+
type GetObjectStreamResult,
|
|
18
|
+
type ObjectIO,
|
|
19
|
+
type PutPrecondition,
|
|
20
|
+
} from "./object-io.js";
|
|
13
21
|
|
|
14
22
|
// Byte/metadata transport is resolved per-call via resolveObjectIO(ctx) — the
|
|
15
23
|
// default is the AWS S3 SDK over STS-vended credentials (S3SdkObjectIO), but a
|
|
@@ -142,6 +150,7 @@ export function decodeSymlinkMetadataValue(value: string): string {
|
|
|
142
150
|
* extension can encode additional fields if needed.
|
|
143
151
|
*/
|
|
144
152
|
export const SYMLINK_BODY_PREFIX = "hq-symlink:";
|
|
153
|
+
const SYMLINK_BODY_PREFIX_BYTES = Buffer.from(SYMLINK_BODY_PREFIX, "utf-8");
|
|
145
154
|
|
|
146
155
|
/**
|
|
147
156
|
* S3 user-metadata key carrying the source-side file mode (permission bits
|
|
@@ -230,6 +239,116 @@ export function encodeSymlinkBody(target: string): Buffer {
|
|
|
230
239
|
return Buffer.from(SYMLINK_BODY_PREFIX + target, "utf-8");
|
|
231
240
|
}
|
|
232
241
|
|
|
242
|
+
function downloadTempPath(localPath: string): string {
|
|
243
|
+
const dir = path.dirname(localPath);
|
|
244
|
+
const random = crypto.randomBytes(10).toString("hex");
|
|
245
|
+
return path.join(dir, `.hq-tmp-${random}`);
|
|
246
|
+
}
|
|
247
|
+
|
|
248
|
+
function removeTempPath(tempPath: string): void {
|
|
249
|
+
try {
|
|
250
|
+
fs.unlinkSync(tempPath);
|
|
251
|
+
} catch {
|
|
252
|
+
// Best-effort cleanup; do not mask the transfer/materialization error.
|
|
253
|
+
}
|
|
254
|
+
}
|
|
255
|
+
|
|
256
|
+
async function getObjectStream(
|
|
257
|
+
io: ObjectIO,
|
|
258
|
+
key: string,
|
|
259
|
+
): Promise<GetObjectStreamResult> {
|
|
260
|
+
if (io.getObjectStream) return io.getObjectStream(key);
|
|
261
|
+
const res = await io.getObject(key);
|
|
262
|
+
return {
|
|
263
|
+
body: (async function* () {
|
|
264
|
+
yield res.body;
|
|
265
|
+
})(),
|
|
266
|
+
metadata: res.metadata,
|
|
267
|
+
};
|
|
268
|
+
}
|
|
269
|
+
|
|
270
|
+
function bufferFromChunk(chunk: Uint8Array): Buffer {
|
|
271
|
+
if (Buffer.isBuffer(chunk)) return chunk;
|
|
272
|
+
return Buffer.from(chunk.buffer, chunk.byteOffset, chunk.byteLength);
|
|
273
|
+
}
|
|
274
|
+
|
|
275
|
+
function leadingBytes(chunks: Buffer[], totalLength: number, length: number): Buffer {
|
|
276
|
+
const out = Buffer.alloc(Math.min(totalLength, length));
|
|
277
|
+
let offset = 0;
|
|
278
|
+
for (const chunk of chunks) {
|
|
279
|
+
if (offset >= out.length) break;
|
|
280
|
+
const take = Math.min(chunk.length, out.length - offset);
|
|
281
|
+
chunk.copy(out, offset, 0, take);
|
|
282
|
+
offset += take;
|
|
283
|
+
}
|
|
284
|
+
return out;
|
|
285
|
+
}
|
|
286
|
+
|
|
287
|
+
async function collectRemainingChunks(
|
|
288
|
+
initialChunks: Buffer[],
|
|
289
|
+
iterator: AsyncIterator<Uint8Array>,
|
|
290
|
+
): Promise<Buffer> {
|
|
291
|
+
const chunks = [...initialChunks];
|
|
292
|
+
let totalLength = chunks.reduce((sum, chunk) => sum + chunk.length, 0);
|
|
293
|
+
while (true) {
|
|
294
|
+
const next = await iterator.next();
|
|
295
|
+
if (next.done) break;
|
|
296
|
+
const chunk = bufferFromChunk(next.value);
|
|
297
|
+
if (chunk.length === 0) continue;
|
|
298
|
+
chunks.push(chunk);
|
|
299
|
+
totalLength += chunk.length;
|
|
300
|
+
}
|
|
301
|
+
return Buffer.concat(chunks, totalLength);
|
|
302
|
+
}
|
|
303
|
+
|
|
304
|
+
async function* regularDownloadChunks(
|
|
305
|
+
initialChunks: Buffer[],
|
|
306
|
+
iterator: AsyncIterator<Uint8Array>,
|
|
307
|
+
): AsyncIterable<Buffer> {
|
|
308
|
+
for (const chunk of initialChunks) {
|
|
309
|
+
if (chunk.length > 0) yield chunk;
|
|
310
|
+
}
|
|
311
|
+
while (true) {
|
|
312
|
+
const next = await iterator.next();
|
|
313
|
+
if (next.done) break;
|
|
314
|
+
const chunk = bufferFromChunk(next.value);
|
|
315
|
+
if (chunk.length > 0) yield chunk;
|
|
316
|
+
}
|
|
317
|
+
}
|
|
318
|
+
|
|
319
|
+
class HashingTransform extends Transform {
|
|
320
|
+
private readonly hash = crypto.createHash("sha256");
|
|
321
|
+
size = 0;
|
|
322
|
+
|
|
323
|
+
_transform(
|
|
324
|
+
chunk: Buffer,
|
|
325
|
+
_encoding: BufferEncoding,
|
|
326
|
+
callback: (error?: Error | null, data?: Buffer) => void,
|
|
327
|
+
): void {
|
|
328
|
+
this.hash.update(chunk);
|
|
329
|
+
this.size += chunk.length;
|
|
330
|
+
callback(null, chunk);
|
|
331
|
+
}
|
|
332
|
+
|
|
333
|
+
digest(): string {
|
|
334
|
+
return this.hash.digest("hex");
|
|
335
|
+
}
|
|
336
|
+
}
|
|
337
|
+
|
|
338
|
+
async function streamRegularFileToTemp(
|
|
339
|
+
tempPath: string,
|
|
340
|
+
initialChunks: Buffer[],
|
|
341
|
+
iterator: AsyncIterator<Uint8Array>,
|
|
342
|
+
): Promise<{ hash: string; size: number }> {
|
|
343
|
+
const hashing = new HashingTransform();
|
|
344
|
+
await pipeline(
|
|
345
|
+
Readable.from(regularDownloadChunks(initialChunks, iterator)),
|
|
346
|
+
hashing,
|
|
347
|
+
fs.createWriteStream(tempPath, { flags: "wx" }),
|
|
348
|
+
);
|
|
349
|
+
return { hash: hashing.digest(), size: hashing.size };
|
|
350
|
+
}
|
|
351
|
+
|
|
233
352
|
/**
|
|
234
353
|
* Batch pre-mint transport URLs for `keys` under `op` so the subsequent
|
|
235
354
|
* per-file transfer calls (downloadFile/headRemoteFile/…) reuse them instead
|
|
@@ -558,15 +677,25 @@ export async function downloadFile(
|
|
|
558
677
|
ctx: EntityContext,
|
|
559
678
|
key: string,
|
|
560
679
|
localPath: string,
|
|
561
|
-
): Promise<{
|
|
680
|
+
): Promise<{
|
|
681
|
+
metadata?: Record<string, string>;
|
|
682
|
+
contentHash?: string;
|
|
683
|
+
contentSize?: number;
|
|
684
|
+
}> {
|
|
562
685
|
const io = resolveObjectIO(ctx);
|
|
563
686
|
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
|
|
687
|
+
const { body, metadata } = await getObjectStream(io, key);
|
|
688
|
+
const iterator = body[Symbol.asyncIterator]();
|
|
689
|
+
const initialChunks: Buffer[] = [];
|
|
690
|
+
let initialLength = 0;
|
|
691
|
+
while (initialLength < SYMLINK_BODY_PREFIX_BYTES.length) {
|
|
692
|
+
const next = await iterator.next();
|
|
693
|
+
if (next.done) break;
|
|
694
|
+
const chunk = bufferFromChunk(next.value);
|
|
695
|
+
if (chunk.length === 0) continue;
|
|
696
|
+
initialChunks.push(chunk);
|
|
697
|
+
initialLength += chunk.length;
|
|
698
|
+
}
|
|
570
699
|
|
|
571
700
|
const dir = path.dirname(localPath);
|
|
572
701
|
if (!fs.existsSync(dir)) {
|
|
@@ -592,18 +721,31 @@ export async function downloadFile(
|
|
|
592
721
|
// The body is already buffered (tiny for symlink records); reading it here
|
|
593
722
|
// is behavior-preserving for regular files (whose body never starts with
|
|
594
723
|
// the prefix per the SYMLINK_BODY_PREFIX doc). See SYMLINK_BODY_PREFIX.
|
|
595
|
-
const
|
|
724
|
+
const leading = leadingBytes(
|
|
725
|
+
initialChunks,
|
|
726
|
+
initialLength,
|
|
727
|
+
SYMLINK_BODY_PREFIX_BYTES.length,
|
|
728
|
+
);
|
|
729
|
+
const bodyHasSymlinkPrefix =
|
|
730
|
+
leading.length >= SYMLINK_BODY_PREFIX_BYTES.length &&
|
|
731
|
+
leading.subarray(0, SYMLINK_BODY_PREFIX_BYTES.length).equals(
|
|
732
|
+
SYMLINK_BODY_PREFIX_BYTES,
|
|
733
|
+
);
|
|
596
734
|
const isSymlinkRecord =
|
|
597
735
|
(typeof symlinkMarker === "string" && symlinkMarker.length > 0) ||
|
|
598
|
-
|
|
736
|
+
bodyHasSymlinkPrefix;
|
|
599
737
|
if (isSymlinkRecord) {
|
|
738
|
+
const objectBody = await collectRemainingChunks(initialChunks, iterator);
|
|
600
739
|
// The target lives in the body (marker-only metadata convention).
|
|
601
740
|
// Symlink record bodies are bounded by target length (typically
|
|
602
741
|
// <300 bytes for relative paths, hard-capped by S3's 5 GB object
|
|
603
|
-
// size); the
|
|
742
|
+
// size); only this branch buffers the body so regular files can stream.
|
|
604
743
|
let symlinkTarget: string;
|
|
605
|
-
if (
|
|
606
|
-
symlinkTarget =
|
|
744
|
+
if (bodyHasSymlinkPrefix) {
|
|
745
|
+
symlinkTarget = objectBody.toString(
|
|
746
|
+
"utf-8",
|
|
747
|
+
SYMLINK_BODY_PREFIX_BYTES.length,
|
|
748
|
+
);
|
|
607
749
|
} else {
|
|
608
750
|
// Backward-compat fallback: a legacy upload from earlier in
|
|
609
751
|
// this PR's lifetime stored the target in metadata (raw or
|
|
@@ -619,127 +761,100 @@ export async function downloadFile(
|
|
|
619
761
|
|
|
620
762
|
if (symlinkTarget.length === 0) {
|
|
621
763
|
throw new Error(
|
|
622
|
-
`Symlink record for ${key} had no target (body: ${
|
|
764
|
+
`Symlink record for ${key} had no target (body: ${objectBody.length} bytes, marker: ${symlinkMarker})`,
|
|
623
765
|
);
|
|
624
766
|
}
|
|
625
767
|
|
|
626
|
-
|
|
627
|
-
// a stale symlink this also frees the slot. ENOENT is fine — first
|
|
628
|
-
// download of this key has nothing to clear. Other errors propagate
|
|
629
|
-
// because they signal a real problem (permissions, parent missing).
|
|
768
|
+
const tempPath = downloadTempPath(localPath);
|
|
630
769
|
try {
|
|
631
|
-
fs.
|
|
632
|
-
|
|
633
|
-
|
|
634
|
-
|
|
635
|
-
|
|
636
|
-
!("code" in err) ||
|
|
637
|
-
(err as { code?: string }).code !== "ENOENT"
|
|
638
|
-
) {
|
|
639
|
-
throw err;
|
|
640
|
-
}
|
|
770
|
+
fs.symlinkSync(symlinkTarget, tempPath);
|
|
771
|
+
fs.renameSync(tempPath, localPath);
|
|
772
|
+
} catch (err) {
|
|
773
|
+
removeTempPath(tempPath);
|
|
774
|
+
throw err;
|
|
641
775
|
}
|
|
642
|
-
fs.symlinkSync(symlinkTarget, localPath);
|
|
643
776
|
return { metadata };
|
|
644
777
|
}
|
|
645
|
-
|
|
646
|
-
|
|
647
|
-
|
|
648
|
-
// path still holds the stale symlink from the last sync. Without this
|
|
649
|
-
// unlink, fs.writeFileSync follows the link and overwrites its
|
|
650
|
-
// target file's contents — leaving the link in place and the new
|
|
651
|
-
// regular object never materializing at the intended path. lstat
|
|
652
|
-
// (not statSync) avoids following the link to test what's there.
|
|
778
|
+
const tempPath = downloadTempPath(localPath);
|
|
779
|
+
let tempReady = true;
|
|
780
|
+
let streamed: { hash: string; size: number };
|
|
653
781
|
try {
|
|
654
|
-
|
|
655
|
-
|
|
656
|
-
|
|
657
|
-
|
|
658
|
-
|
|
659
|
-
//
|
|
660
|
-
|
|
661
|
-
|
|
662
|
-
|
|
663
|
-
|
|
664
|
-
|
|
665
|
-
|
|
666
|
-
|
|
667
|
-
|
|
668
|
-
|
|
669
|
-
|
|
670
|
-
|
|
671
|
-
|
|
672
|
-
|
|
673
|
-
|
|
674
|
-
|
|
675
|
-
|
|
676
|
-
|
|
677
|
-
|
|
678
|
-
|
|
679
|
-
|
|
680
|
-
|
|
681
|
-
// parses to 0o755 instead of NaN — so tampered or malformed metadata
|
|
682
|
-
// could still change local permissions unexpectedly. The regex
|
|
683
|
-
// requires 1–4 pure octal digits (`[0-7]{1,4}$`), which matches what
|
|
684
|
-
// the upload side stamps (`(mode & 0o777).toString(8)` → at most
|
|
685
|
-
// three digits, all 0–7) and rejects everything else.
|
|
686
|
-
const modeOctal = metadata?.[FILE_MODE_META_KEY];
|
|
687
|
-
if (typeof modeOctal === "string" && /^[0-7]{1,4}$/.test(modeOctal)) {
|
|
688
|
-
const parsed = parseInt(modeOctal, 8);
|
|
689
|
-
if (Number.isFinite(parsed) && parsed >= 0 && parsed <= 0o777) {
|
|
690
|
-
try {
|
|
691
|
-
fs.chmodSync(localPath, parsed);
|
|
692
|
-
} catch {
|
|
693
|
-
// chmod failure (read-only FS, EPERM) is non-fatal — the file
|
|
694
|
-
// is materialized, just with the umask default. Surface via
|
|
695
|
-
// S3-side metadata being present but the file not matching;
|
|
696
|
-
// a future operator-side audit can reconcile.
|
|
782
|
+
streamed = await streamRegularFileToTemp(tempPath, initialChunks, iterator);
|
|
783
|
+
|
|
784
|
+
// Bug #5 — apply source-side mode after the byte write. See
|
|
785
|
+
// FILE_MODE_META_KEY for the metadata contract. Parses defensively:
|
|
786
|
+
// a malformed value falls through with no chmod so the umask default
|
|
787
|
+
// applies, matching the legacy back-compat path. The staged path is a
|
|
788
|
+
// regular file, then it is atomically renamed over the destination.
|
|
789
|
+
//
|
|
790
|
+
// Codex P2 (PR #24 round 3): strict octal-only regex BEFORE parseInt.
|
|
791
|
+
// parseInt(modeOctal, 8) accepts partial-prefix garbage — "755junk"
|
|
792
|
+
// parses to 0o755 instead of NaN — so tampered or malformed metadata
|
|
793
|
+
// could still change local permissions unexpectedly. The regex
|
|
794
|
+
// requires 1–4 pure octal digits (`[0-7]{1,4}$`), which matches what
|
|
795
|
+
// the upload side stamps (`(mode & 0o777).toString(8)` → at most
|
|
796
|
+
// three digits, all 0–7) and rejects everything else.
|
|
797
|
+
const modeOctal = metadata?.[FILE_MODE_META_KEY];
|
|
798
|
+
if (typeof modeOctal === "string" && /^[0-7]{1,4}$/.test(modeOctal)) {
|
|
799
|
+
const parsed = parseInt(modeOctal, 8);
|
|
800
|
+
if (Number.isFinite(parsed) && parsed >= 0 && parsed <= 0o777) {
|
|
801
|
+
try {
|
|
802
|
+
fs.chmodSync(tempPath, parsed);
|
|
803
|
+
} catch {
|
|
804
|
+
// chmod failure (read-only FS, EPERM) is non-fatal — the file
|
|
805
|
+
// is materialized, just with the umask default. Surface via
|
|
806
|
+
// S3-side metadata being present but the file not matching;
|
|
807
|
+
// a future operator-side audit can reconcile.
|
|
808
|
+
}
|
|
697
809
|
}
|
|
698
810
|
}
|
|
699
|
-
}
|
|
700
811
|
|
|
701
|
-
|
|
702
|
-
|
|
703
|
-
|
|
704
|
-
|
|
705
|
-
|
|
706
|
-
|
|
707
|
-
|
|
708
|
-
|
|
709
|
-
|
|
710
|
-
|
|
711
|
-
|
|
712
|
-
|
|
713
|
-
|
|
714
|
-
|
|
715
|
-
|
|
716
|
-
|
|
717
|
-
|
|
718
|
-
|
|
719
|
-
|
|
720
|
-
|
|
721
|
-
|
|
722
|
-
|
|
723
|
-
|
|
724
|
-
|
|
725
|
-
|
|
726
|
-
|
|
727
|
-
|
|
728
|
-
|
|
729
|
-
|
|
730
|
-
|
|
731
|
-
|
|
732
|
-
|
|
733
|
-
|
|
734
|
-
|
|
735
|
-
|
|
736
|
-
|
|
737
|
-
|
|
738
|
-
|
|
739
|
-
|
|
740
|
-
// can re-sync next pass.
|
|
812
|
+
// 5.37.0 — apply source-side mtime after the byte write (and after the
|
|
813
|
+
// chmod above; ordering between chmod and utimes doesn't matter, but
|
|
814
|
+
// both must run AFTER writeFileSync because writeFileSync resets mtime
|
|
815
|
+
// to wall-clock-now). See FILE_MTIME_META_KEY for the metadata contract.
|
|
816
|
+
//
|
|
817
|
+
// Strict-numeric regex BEFORE parseInt — same Codex P2 lesson as hq-mode.
|
|
818
|
+
// `^-?[0-9]{1,16}$` rejects partial-prefix garbage ("175junk" → 175),
|
|
819
|
+
// empty, double-signed, decimals, whitespace, and oversized strings. A
|
|
820
|
+
// single optional leading `-` is allowed so pre-epoch / reproducible-build
|
|
821
|
+
// timestamps round-trip (Codex PR #27 P2 — `mtimeMs === 0` and negative
|
|
822
|
+
// epoch values are legitimate). 16 digits comfortably covers any plausible
|
|
823
|
+
// epoch-ms value (year ~5138 is 16 digits; we'll cross that bridge later).
|
|
824
|
+
//
|
|
825
|
+
// The staged file is regular; rename preserves these stamps at localPath.
|
|
826
|
+
//
|
|
827
|
+
// Composition with the 5.36.0 lstat fast-path: the journal stamp at the
|
|
828
|
+
// share/sync call sites runs AFTER downloadFile returns, so the lstat
|
|
829
|
+
// it captures sees the post-utimes mtime. Verified in cli/sync.ts
|
|
830
|
+
// (downloadFile → lstatSync → updateEntry) and cli/share.ts (pull path
|
|
831
|
+
// similarly lstats after downloadFile). If a future caller stamps the
|
|
832
|
+
// journal BEFORE downloadFile completes, the fast-path will stale and
|
|
833
|
+
// re-hash every sync forever — keep the call-site invariant intact.
|
|
834
|
+
const mtimeRaw = metadata?.[FILE_MTIME_META_KEY];
|
|
835
|
+
if (typeof mtimeRaw === "string" && /^-?[0-9]{1,16}$/.test(mtimeRaw)) {
|
|
836
|
+
const mtimeMs = parseInt(mtimeRaw, 10);
|
|
837
|
+
if (Number.isFinite(mtimeMs)) {
|
|
838
|
+
try {
|
|
839
|
+
// utimesSync accepts seconds OR Date; use Date(ms) for precision.
|
|
840
|
+
// atime = mtime is fine — many filesystems are mounted noatime
|
|
841
|
+
// and distinguishing "access" vs "modification" time doesn't
|
|
842
|
+
// matter for sync semantics. Setting both keeps the on-disk
|
|
843
|
+
// state deterministic across receivers.
|
|
844
|
+
const mtimeDate = new Date(mtimeMs);
|
|
845
|
+
fs.utimesSync(tempPath, mtimeDate, mtimeDate);
|
|
846
|
+
} catch {
|
|
847
|
+
// EPERM / read-only FS / file just unlinked → non-fatal. The
|
|
848
|
+
// file is materialized at write-time mtime; the source-of-truth
|
|
849
|
+
// can re-sync next pass.
|
|
850
|
+
}
|
|
741
851
|
}
|
|
742
852
|
}
|
|
853
|
+
|
|
854
|
+
fs.renameSync(tempPath, localPath);
|
|
855
|
+
tempReady = false;
|
|
856
|
+
} finally {
|
|
857
|
+
if (tempReady) removeTempPath(tempPath);
|
|
743
858
|
}
|
|
744
859
|
|
|
745
860
|
// TODO: stamp hq-btime once Node lands lbirthtime (or birthtimeSync).
|
|
@@ -747,7 +862,7 @@ export async function downloadFile(
|
|
|
747
862
|
// distinct creation time, so a future receiver upgrade picks it up
|
|
748
863
|
// automatically without a server-side data migration.
|
|
749
864
|
|
|
750
|
-
return { metadata };
|
|
865
|
+
return { metadata, contentHash: streamed.hash, contentSize: streamed.size };
|
|
751
866
|
}
|
|
752
867
|
|
|
753
868
|
export interface RemoteFile {
|
package/src/scope-shrink.ts
CHANGED
|
@@ -28,7 +28,10 @@ import type {
|
|
|
28
28
|
SyncJournal,
|
|
29
29
|
} from "./types.js";
|
|
30
30
|
import { hashFile, tombstoneEntry } from "./journal.js";
|
|
31
|
-
import {
|
|
31
|
+
import {
|
|
32
|
+
isCoveredByAny,
|
|
33
|
+
type ScopePrefixInput,
|
|
34
|
+
} from "./prefix-coalesce.js";
|
|
32
35
|
|
|
33
36
|
export interface OrphanClassification {
|
|
34
37
|
/** Relative path (journal key). */
|
|
@@ -59,9 +62,9 @@ export interface BuildScopeShrinkPlanInput {
|
|
|
59
62
|
journal: SyncJournal;
|
|
60
63
|
hqRoot: string;
|
|
61
64
|
/** Coalesced prefixes used by the LAST pull for this company. */
|
|
62
|
-
lastPrefixSet:
|
|
65
|
+
lastPrefixSet: readonly ScopePrefixInput[];
|
|
63
66
|
/** Coalesced prefixes the CURRENT pull will use. */
|
|
64
|
-
currentPrefixSet:
|
|
67
|
+
currentPrefixSet: readonly ScopePrefixInput[];
|
|
65
68
|
/**
|
|
66
69
|
* The caller's own Cognito `sub`. When set, a file the caller authored
|
|
67
70
|
* (`entry.createdBySub === callerSub`) is NEVER orphaned by a scope shrink —
|
|
@@ -536,6 +536,115 @@ describe("collectAndSendSkillTelemetry — hqRoot scoping", () => {
|
|
|
536
536
|
await fs.rm(tmp, { recursive: true, force: true });
|
|
537
537
|
});
|
|
538
538
|
|
|
539
|
+
it("R-F19: bounds a single oversized sanitized skill row before POST", async () => {
|
|
540
|
+
const tmp = await fs.mkdtemp(path.join(os.tmpdir(), "skill-tel-r-f19-"));
|
|
541
|
+
const projects = path.join(tmp, "projects");
|
|
542
|
+
const dir = path.join(projects, "-p");
|
|
543
|
+
await fs.mkdir(dir, { recursive: true });
|
|
544
|
+
const hugeSkill = `deploy-${"x".repeat(400_000)}`;
|
|
545
|
+
await fs.writeFile(
|
|
546
|
+
path.join(dir, "s.jsonl"),
|
|
547
|
+
row({
|
|
548
|
+
type: "user",
|
|
549
|
+
sessionId: "s-r-f19-singleton",
|
|
550
|
+
timestamp: "2026-06-19T11:00:00.000Z",
|
|
551
|
+
cwd: "/x",
|
|
552
|
+
uuid: "u-r-f19-singleton",
|
|
553
|
+
message: {
|
|
554
|
+
role: "user",
|
|
555
|
+
content: `<command-name>/${hugeSkill}</command-name>`,
|
|
556
|
+
},
|
|
557
|
+
}) + "\n",
|
|
558
|
+
"utf-8",
|
|
559
|
+
);
|
|
560
|
+
|
|
561
|
+
const captured: SkillInvocationBatch[] = [];
|
|
562
|
+
const logs: string[] = [];
|
|
563
|
+
const result = await collectAndSendSkillTelemetry({
|
|
564
|
+
client: stubClient(captured),
|
|
565
|
+
machineId: "m",
|
|
566
|
+
installerVersion: "t",
|
|
567
|
+
claudeProjectsRoot: projects,
|
|
568
|
+
codexSessionsRoot: path.join(tmp, "codex"),
|
|
569
|
+
cursorPath: path.join(tmp, "cursor.json"),
|
|
570
|
+
log: (msg) => logs.push(msg),
|
|
571
|
+
});
|
|
572
|
+
|
|
573
|
+
expect(result.eventsSent).toBe(1);
|
|
574
|
+
expect(captured).toHaveLength(1);
|
|
575
|
+
const wireSize = Buffer.byteLength(JSON.stringify(captured[0]), "utf-8");
|
|
576
|
+
expect(wireSize).toBeLessThanOrEqual(240 * 1024);
|
|
577
|
+
expect(captured[0].events[0].skill).not.toBe(hugeSkill);
|
|
578
|
+
expect(logs.some((line) => line.includes("oversized row truncated"))).toBe(
|
|
579
|
+
true,
|
|
580
|
+
);
|
|
581
|
+
|
|
582
|
+
await fs.rm(tmp, { recursive: true, force: true });
|
|
583
|
+
});
|
|
584
|
+
|
|
585
|
+
it("R-F19: a failed earlier skill batch prevents cursor advancement past it", async () => {
|
|
586
|
+
const tmp = await fs.mkdtemp(path.join(os.tmpdir(), "skill-tel-r-f19-"));
|
|
587
|
+
const projects = path.join(tmp, "projects");
|
|
588
|
+
const dir = path.join(projects, "-p");
|
|
589
|
+
await fs.mkdir(dir, { recursive: true });
|
|
590
|
+
const file = path.join(dir, "s.jsonl");
|
|
591
|
+
await fs.writeFile(
|
|
592
|
+
file,
|
|
593
|
+
Array.from({ length: 101 }, (_, i) =>
|
|
594
|
+
row({
|
|
595
|
+
type: "user",
|
|
596
|
+
sessionId: "s-r-f19-prefix",
|
|
597
|
+
timestamp: `2026-06-19T12:${String(Math.floor(i / 60)).padStart(2, "0")}:${String(i % 60).padStart(2, "0")}.000Z`,
|
|
598
|
+
cwd: "/x",
|
|
599
|
+
uuid: `u-r-f19-${i}`,
|
|
600
|
+
message: {
|
|
601
|
+
role: "user",
|
|
602
|
+
content: `<command-name>/skill-${i}</command-name>`,
|
|
603
|
+
},
|
|
604
|
+
}),
|
|
605
|
+
).join("\n") + "\n",
|
|
606
|
+
"utf-8",
|
|
607
|
+
);
|
|
608
|
+
const cursorPath = path.join(tmp, "cursor.json");
|
|
609
|
+
const captured: SkillInvocationBatch[] = [];
|
|
610
|
+
let calls = 0;
|
|
611
|
+
const client = {
|
|
612
|
+
async getTelemetryOptIn() {
|
|
613
|
+
return { enabled: true, updatedAt: null };
|
|
614
|
+
},
|
|
615
|
+
async postSkillInvocations(batch: SkillInvocationBatch) {
|
|
616
|
+
calls++;
|
|
617
|
+
if (calls === 1) throw new Error("first batch failed");
|
|
618
|
+
captured.push(batch);
|
|
619
|
+
return { ok: true, written: batch.events.length, skipped: [] };
|
|
620
|
+
},
|
|
621
|
+
};
|
|
622
|
+
const opts = {
|
|
623
|
+
client,
|
|
624
|
+
machineId: "m",
|
|
625
|
+
installerVersion: "t",
|
|
626
|
+
claudeProjectsRoot: projects,
|
|
627
|
+
codexSessionsRoot: path.join(tmp, "codex"),
|
|
628
|
+
cursorPath,
|
|
629
|
+
};
|
|
630
|
+
|
|
631
|
+
const first = await collectAndSendSkillTelemetry(opts);
|
|
632
|
+
expect(first.eventsSent).toBe(0);
|
|
633
|
+
expect(calls).toBe(1);
|
|
634
|
+
const cursorAfterFailure = JSON.parse(
|
|
635
|
+
await fs.readFile(cursorPath, "utf-8"),
|
|
636
|
+
) as { files: Record<string, { offset: number }> };
|
|
637
|
+
expect(cursorAfterFailure.files[file]?.offset ?? 0).toBe(0);
|
|
638
|
+
|
|
639
|
+
const second = await collectAndSendSkillTelemetry(opts);
|
|
640
|
+
expect(second.eventsSent).toBe(101);
|
|
641
|
+
expect(captured.flatMap((b) => b.events.map((e) => e.skill))).toContain(
|
|
642
|
+
"skill-0",
|
|
643
|
+
);
|
|
644
|
+
|
|
645
|
+
await fs.rm(tmp, { recursive: true, force: true });
|
|
646
|
+
});
|
|
647
|
+
|
|
539
648
|
it("picks up only newly-appended events on a later run (incremental offset)", async () => {
|
|
540
649
|
const tmp = await fs.mkdtemp(path.join(os.tmpdir(), "skill-tel-"));
|
|
541
650
|
const projects = path.join(tmp, "projects");
|