@indigoai-us/hq-cloud 6.11.11 → 6.11.13

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (220) hide show
  1. package/dist/bin/sync-runner-company.d.ts +35 -0
  2. package/dist/bin/sync-runner-company.d.ts.map +1 -0
  3. package/dist/bin/sync-runner-company.js +290 -0
  4. package/dist/bin/sync-runner-company.js.map +1 -0
  5. package/dist/bin/sync-runner-events.d.ts +12 -0
  6. package/dist/bin/sync-runner-events.d.ts.map +1 -0
  7. package/dist/bin/sync-runner-events.js +12 -0
  8. package/dist/bin/sync-runner-events.js.map +1 -0
  9. package/dist/bin/sync-runner-planning.d.ts +53 -0
  10. package/dist/bin/sync-runner-planning.d.ts.map +1 -0
  11. package/dist/bin/sync-runner-planning.js +59 -0
  12. package/dist/bin/sync-runner-planning.js.map +1 -0
  13. package/dist/bin/sync-runner-rollup.d.ts +24 -0
  14. package/dist/bin/sync-runner-rollup.d.ts.map +1 -0
  15. package/dist/bin/sync-runner-rollup.js +46 -0
  16. package/dist/bin/sync-runner-rollup.js.map +1 -0
  17. package/dist/bin/sync-runner-telemetry.d.ts +5 -0
  18. package/dist/bin/sync-runner-telemetry.d.ts.map +1 -0
  19. package/dist/bin/sync-runner-telemetry.js +5 -0
  20. package/dist/bin/sync-runner-telemetry.js.map +1 -0
  21. package/dist/bin/sync-runner-watch-loop.d.ts +17 -0
  22. package/dist/bin/sync-runner-watch-loop.d.ts.map +1 -0
  23. package/dist/bin/sync-runner-watch-loop.js +372 -0
  24. package/dist/bin/sync-runner-watch-loop.js.map +1 -0
  25. package/dist/bin/sync-runner-watch-routes.d.ts +25 -0
  26. package/dist/bin/sync-runner-watch-routes.d.ts.map +1 -0
  27. package/dist/bin/sync-runner-watch-routes.js +74 -0
  28. package/dist/bin/sync-runner-watch-routes.js.map +1 -0
  29. package/dist/bin/sync-runner.d.ts +5 -54
  30. package/dist/bin/sync-runner.d.ts.map +1 -1
  31. package/dist/bin/sync-runner.js +76 -978
  32. package/dist/bin/sync-runner.js.map +1 -1
  33. package/dist/bin/sync-runner.test.js +265 -11
  34. package/dist/bin/sync-runner.test.js.map +1 -1
  35. package/dist/cli/reindex.d.ts.map +1 -1
  36. package/dist/cli/reindex.js +34 -17
  37. package/dist/cli/reindex.js.map +1 -1
  38. package/dist/cli/reindex.test.js +39 -5
  39. package/dist/cli/reindex.test.js.map +1 -1
  40. package/dist/cli/rescue-classify-ordering.test.js +75 -0
  41. package/dist/cli/rescue-classify-ordering.test.js.map +1 -1
  42. package/dist/cli/rescue-core.d.ts +45 -0
  43. package/dist/cli/rescue-core.d.ts.map +1 -1
  44. package/dist/cli/rescue-core.js +320 -170
  45. package/dist/cli/rescue-core.js.map +1 -1
  46. package/dist/cli/share.d.ts +2 -1
  47. package/dist/cli/share.d.ts.map +1 -1
  48. package/dist/cli/share.js +276 -660
  49. package/dist/cli/share.js.map +1 -1
  50. package/dist/cli/share.test.js +30 -0
  51. package/dist/cli/share.test.js.map +1 -1
  52. package/dist/cli/sync.d.ts +28 -1
  53. package/dist/cli/sync.d.ts.map +1 -1
  54. package/dist/cli/sync.js +541 -748
  55. package/dist/cli/sync.js.map +1 -1
  56. package/dist/cli/sync.test.js +382 -1
  57. package/dist/cli/sync.test.js.map +1 -1
  58. package/dist/cognito-auth.d.ts.map +1 -1
  59. package/dist/cognito-auth.js +55 -10
  60. package/dist/cognito-auth.js.map +1 -1
  61. package/dist/cognito-auth.test.js +61 -0
  62. package/dist/cognito-auth.test.js.map +1 -1
  63. package/dist/daemon-worker.d.ts +2 -2
  64. package/dist/daemon-worker.js +3 -3
  65. package/dist/daemon-worker.js.map +1 -1
  66. package/dist/index.d.ts +2 -1
  67. package/dist/index.d.ts.map +1 -1
  68. package/dist/index.js +1 -1
  69. package/dist/index.js.map +1 -1
  70. package/dist/journal.d.ts.map +1 -1
  71. package/dist/journal.js +93 -6
  72. package/dist/journal.js.map +1 -1
  73. package/dist/journal.test.js +59 -0
  74. package/dist/journal.test.js.map +1 -1
  75. package/dist/machine-auth.test.js +60 -2
  76. package/dist/machine-auth.test.js.map +1 -1
  77. package/dist/object-io.d.ts +37 -1
  78. package/dist/object-io.d.ts.map +1 -1
  79. package/dist/object-io.js +149 -30
  80. package/dist/object-io.js.map +1 -1
  81. package/dist/object-io.test.js +121 -0
  82. package/dist/object-io.test.js.map +1 -1
  83. package/dist/operation-lock.d.ts +8 -8
  84. package/dist/operation-lock.d.ts.map +1 -1
  85. package/dist/operation-lock.js +99 -32
  86. package/dist/operation-lock.js.map +1 -1
  87. package/dist/operation-lock.test.js +51 -4
  88. package/dist/operation-lock.test.js.map +1 -1
  89. package/dist/personal-vault.d.ts.map +1 -1
  90. package/dist/personal-vault.js +8 -2
  91. package/dist/personal-vault.js.map +1 -1
  92. package/dist/personal-vault.test.js +34 -0
  93. package/dist/personal-vault.test.js.map +1 -1
  94. package/dist/prefix-coalesce.d.ts +20 -9
  95. package/dist/prefix-coalesce.d.ts.map +1 -1
  96. package/dist/prefix-coalesce.js +124 -28
  97. package/dist/prefix-coalesce.js.map +1 -1
  98. package/dist/prefix-coalesce.test.js +57 -2
  99. package/dist/prefix-coalesce.test.js.map +1 -1
  100. package/dist/remote-pull.d.ts +8 -3
  101. package/dist/remote-pull.d.ts.map +1 -1
  102. package/dist/remote-pull.js +85 -16
  103. package/dist/remote-pull.js.map +1 -1
  104. package/dist/remote-pull.test.js +213 -2
  105. package/dist/remote-pull.test.js.map +1 -1
  106. package/dist/s3.d.ts +2 -0
  107. package/dist/s3.d.ts.map +1 -1
  108. package/dist/s3.js +197 -116
  109. package/dist/s3.js.map +1 -1
  110. package/dist/s3.test.js +109 -0
  111. package/dist/s3.test.js.map +1 -1
  112. package/dist/scope-shrink.d.ts +3 -2
  113. package/dist/scope-shrink.d.ts.map +1 -1
  114. package/dist/scope-shrink.js +1 -1
  115. package/dist/scope-shrink.js.map +1 -1
  116. package/dist/skill-telemetry.d.ts +1 -1
  117. package/dist/skill-telemetry.d.ts.map +1 -1
  118. package/dist/skill-telemetry.js +69 -9
  119. package/dist/skill-telemetry.js.map +1 -1
  120. package/dist/skill-telemetry.test.js +86 -0
  121. package/dist/skill-telemetry.test.js.map +1 -1
  122. package/dist/sync/event-sync.d.ts +6 -0
  123. package/dist/sync/event-sync.d.ts.map +1 -1
  124. package/dist/sync/event-sync.js +34 -1
  125. package/dist/sync/event-sync.js.map +1 -1
  126. package/dist/sync/event-sync.test.js +73 -0
  127. package/dist/sync/event-sync.test.js.map +1 -1
  128. package/dist/sync/metrics.d.ts +17 -1
  129. package/dist/sync/metrics.d.ts.map +1 -1
  130. package/dist/sync/metrics.js +32 -1
  131. package/dist/sync/metrics.js.map +1 -1
  132. package/dist/sync/metrics.test.js +74 -1
  133. package/dist/sync/metrics.test.js.map +1 -1
  134. package/dist/sync/pull-scope.d.ts.map +1 -1
  135. package/dist/sync/pull-scope.js +15 -7
  136. package/dist/sync/pull-scope.js.map +1 -1
  137. package/dist/sync/push-receiver.d.ts +12 -5
  138. package/dist/sync/push-receiver.d.ts.map +1 -1
  139. package/dist/sync/push-receiver.js +45 -17
  140. package/dist/sync/push-receiver.js.map +1 -1
  141. package/dist/sync/push-receiver.test.js +67 -1
  142. package/dist/sync/push-receiver.test.js.map +1 -1
  143. package/dist/sync-core.d.ts +27 -0
  144. package/dist/sync-core.d.ts.map +1 -0
  145. package/dist/sync-core.js +54 -0
  146. package/dist/sync-core.js.map +1 -0
  147. package/dist/telemetry.d.ts +1 -1
  148. package/dist/telemetry.d.ts.map +1 -1
  149. package/dist/telemetry.js +59 -6
  150. package/dist/telemetry.js.map +1 -1
  151. package/dist/telemetry.test.js +74 -0
  152. package/dist/telemetry.test.js.map +1 -1
  153. package/dist/types.d.ts +8 -0
  154. package/dist/types.d.ts.map +1 -1
  155. package/dist/vault-client.d.ts.map +1 -1
  156. package/dist/vault-client.js +284 -36
  157. package/dist/vault-client.js.map +1 -1
  158. package/dist/vault-client.test.js +59 -0
  159. package/dist/vault-client.test.js.map +1 -1
  160. package/dist/watcher.d.ts +38 -20
  161. package/dist/watcher.d.ts.map +1 -1
  162. package/dist/watcher.js +155 -143
  163. package/dist/watcher.js.map +1 -1
  164. package/dist/watcher.test.js +103 -0
  165. package/dist/watcher.test.js.map +1 -1
  166. package/package.json +1 -1
  167. package/src/bin/sync-runner-company.ts +350 -0
  168. package/src/bin/sync-runner-events.ts +25 -0
  169. package/src/bin/sync-runner-planning.ts +121 -0
  170. package/src/bin/sync-runner-rollup.ts +72 -0
  171. package/src/bin/sync-runner-telemetry.ts +8 -0
  172. package/src/bin/sync-runner-watch-loop.ts +443 -0
  173. package/src/bin/sync-runner-watch-routes.ts +86 -0
  174. package/src/bin/sync-runner.test.ts +298 -11
  175. package/src/bin/sync-runner.ts +99 -1054
  176. package/src/cli/reindex.test.ts +41 -3
  177. package/src/cli/reindex.ts +35 -19
  178. package/src/cli/rescue-classify-ordering.test.ts +81 -0
  179. package/src/cli/rescue-core.ts +400 -165
  180. package/src/cli/share.test.ts +38 -0
  181. package/src/cli/share.ts +420 -693
  182. package/src/cli/sync.test.ts +460 -1
  183. package/src/cli/sync.ts +788 -825
  184. package/src/cognito-auth.test.ts +77 -0
  185. package/src/cognito-auth.ts +73 -11
  186. package/src/daemon-worker.ts +3 -3
  187. package/src/index.ts +8 -0
  188. package/src/journal.test.ts +72 -0
  189. package/src/journal.ts +95 -8
  190. package/src/machine-auth.test.ts +64 -2
  191. package/src/object-io.test.ts +142 -0
  192. package/src/object-io.ts +183 -31
  193. package/src/operation-lock.test.ts +63 -4
  194. package/src/operation-lock.ts +99 -31
  195. package/src/personal-vault.test.ts +42 -0
  196. package/src/personal-vault.ts +8 -2
  197. package/src/prefix-coalesce.test.ts +71 -1
  198. package/src/prefix-coalesce.ts +155 -30
  199. package/src/remote-pull.test.ts +235 -1
  200. package/src/remote-pull.ts +106 -18
  201. package/src/s3.test.ts +126 -0
  202. package/src/s3.ts +237 -122
  203. package/src/scope-shrink.ts +6 -3
  204. package/src/skill-telemetry.test.ts +109 -0
  205. package/src/skill-telemetry.ts +82 -14
  206. package/src/sync/event-sync.test.ts +75 -0
  207. package/src/sync/event-sync.ts +54 -1
  208. package/src/sync/metrics.test.ts +81 -0
  209. package/src/sync/metrics.ts +59 -4
  210. package/src/sync/pull-scope.ts +23 -7
  211. package/src/sync/push-receiver.test.ts +73 -1
  212. package/src/sync/push-receiver.ts +56 -20
  213. package/src/sync-core.ts +58 -0
  214. package/src/telemetry.test.ts +85 -0
  215. package/src/telemetry.ts +69 -6
  216. package/src/types.ts +8 -0
  217. package/src/vault-client.test.ts +74 -0
  218. package/src/vault-client.ts +395 -43
  219. package/src/watcher.test.ts +117 -0
  220. package/src/watcher.ts +215 -174
package/src/s3.ts CHANGED
@@ -8,8 +8,16 @@
8
8
 
9
9
  import * as fs from "fs";
10
10
  import * as path from "path";
11
+ import * as crypto from "crypto";
12
+ import { Readable, Transform } from "stream";
13
+ import { pipeline } from "stream/promises";
11
14
  import type { EntityContext } from "./types.js";
12
- import { resolveObjectIO, type ObjectIO, type PutPrecondition } from "./object-io.js";
15
+ import {
16
+ resolveObjectIO,
17
+ type GetObjectStreamResult,
18
+ type ObjectIO,
19
+ type PutPrecondition,
20
+ } from "./object-io.js";
13
21
 
14
22
  // Byte/metadata transport is resolved per-call via resolveObjectIO(ctx) — the
15
23
  // default is the AWS S3 SDK over STS-vended credentials (S3SdkObjectIO), but a
@@ -142,6 +150,7 @@ export function decodeSymlinkMetadataValue(value: string): string {
142
150
  * extension can encode additional fields if needed.
143
151
  */
144
152
  export const SYMLINK_BODY_PREFIX = "hq-symlink:";
153
+ const SYMLINK_BODY_PREFIX_BYTES = Buffer.from(SYMLINK_BODY_PREFIX, "utf-8");
145
154
 
146
155
  /**
147
156
  * S3 user-metadata key carrying the source-side file mode (permission bits
@@ -230,6 +239,116 @@ export function encodeSymlinkBody(target: string): Buffer {
230
239
  return Buffer.from(SYMLINK_BODY_PREFIX + target, "utf-8");
231
240
  }
232
241
 
242
+ function downloadTempPath(localPath: string): string {
243
+ const dir = path.dirname(localPath);
244
+ const random = crypto.randomBytes(10).toString("hex");
245
+ return path.join(dir, `.hq-tmp-${random}`);
246
+ }
247
+
248
+ function removeTempPath(tempPath: string): void {
249
+ try {
250
+ fs.unlinkSync(tempPath);
251
+ } catch {
252
+ // Best-effort cleanup; do not mask the transfer/materialization error.
253
+ }
254
+ }
255
+
256
+ async function getObjectStream(
257
+ io: ObjectIO,
258
+ key: string,
259
+ ): Promise<GetObjectStreamResult> {
260
+ if (io.getObjectStream) return io.getObjectStream(key);
261
+ const res = await io.getObject(key);
262
+ return {
263
+ body: (async function* () {
264
+ yield res.body;
265
+ })(),
266
+ metadata: res.metadata,
267
+ };
268
+ }
269
+
270
+ function bufferFromChunk(chunk: Uint8Array): Buffer {
271
+ if (Buffer.isBuffer(chunk)) return chunk;
272
+ return Buffer.from(chunk.buffer, chunk.byteOffset, chunk.byteLength);
273
+ }
274
+
275
+ function leadingBytes(chunks: Buffer[], totalLength: number, length: number): Buffer {
276
+ const out = Buffer.alloc(Math.min(totalLength, length));
277
+ let offset = 0;
278
+ for (const chunk of chunks) {
279
+ if (offset >= out.length) break;
280
+ const take = Math.min(chunk.length, out.length - offset);
281
+ chunk.copy(out, offset, 0, take);
282
+ offset += take;
283
+ }
284
+ return out;
285
+ }
286
+
287
+ async function collectRemainingChunks(
288
+ initialChunks: Buffer[],
289
+ iterator: AsyncIterator<Uint8Array>,
290
+ ): Promise<Buffer> {
291
+ const chunks = [...initialChunks];
292
+ let totalLength = chunks.reduce((sum, chunk) => sum + chunk.length, 0);
293
+ while (true) {
294
+ const next = await iterator.next();
295
+ if (next.done) break;
296
+ const chunk = bufferFromChunk(next.value);
297
+ if (chunk.length === 0) continue;
298
+ chunks.push(chunk);
299
+ totalLength += chunk.length;
300
+ }
301
+ return Buffer.concat(chunks, totalLength);
302
+ }
303
+
304
+ async function* regularDownloadChunks(
305
+ initialChunks: Buffer[],
306
+ iterator: AsyncIterator<Uint8Array>,
307
+ ): AsyncIterable<Buffer> {
308
+ for (const chunk of initialChunks) {
309
+ if (chunk.length > 0) yield chunk;
310
+ }
311
+ while (true) {
312
+ const next = await iterator.next();
313
+ if (next.done) break;
314
+ const chunk = bufferFromChunk(next.value);
315
+ if (chunk.length > 0) yield chunk;
316
+ }
317
+ }
318
+
319
+ class HashingTransform extends Transform {
320
+ private readonly hash = crypto.createHash("sha256");
321
+ size = 0;
322
+
323
+ _transform(
324
+ chunk: Buffer,
325
+ _encoding: BufferEncoding,
326
+ callback: (error?: Error | null, data?: Buffer) => void,
327
+ ): void {
328
+ this.hash.update(chunk);
329
+ this.size += chunk.length;
330
+ callback(null, chunk);
331
+ }
332
+
333
+ digest(): string {
334
+ return this.hash.digest("hex");
335
+ }
336
+ }
337
+
338
+ async function streamRegularFileToTemp(
339
+ tempPath: string,
340
+ initialChunks: Buffer[],
341
+ iterator: AsyncIterator<Uint8Array>,
342
+ ): Promise<{ hash: string; size: number }> {
343
+ const hashing = new HashingTransform();
344
+ await pipeline(
345
+ Readable.from(regularDownloadChunks(initialChunks, iterator)),
346
+ hashing,
347
+ fs.createWriteStream(tempPath, { flags: "wx" }),
348
+ );
349
+ return { hash: hashing.digest(), size: hashing.size };
350
+ }
351
+
233
352
  /**
234
353
  * Batch pre-mint transport URLs for `keys` under `op` so the subsequent
235
354
  * per-file transfer calls (downloadFile/headRemoteFile/…) reuse them instead
@@ -558,15 +677,25 @@ export async function downloadFile(
558
677
  ctx: EntityContext,
559
678
  key: string,
560
679
  localPath: string,
561
- ): Promise<{ metadata?: Record<string, string> }> {
680
+ ): Promise<{
681
+ metadata?: Record<string, string>;
682
+ contentHash?: string;
683
+ contentSize?: number;
684
+ }> {
562
685
  const io = resolveObjectIO(ctx);
563
686
 
564
- // The transport returns the full object body buffered + its user metadata.
565
- // downloadFile already buffered the whole object (writeFileSync of the
566
- // concatenated chunks), so buffering at the transport layer is behavior-
567
- // preserving symlink record bodies are tiny and regular files were read
568
- // fully into memory regardless.
569
- const { body: objectBody, metadata } = await io.getObject(key);
687
+ const { body, metadata } = await getObjectStream(io, key);
688
+ const iterator = body[Symbol.asyncIterator]();
689
+ const initialChunks: Buffer[] = [];
690
+ let initialLength = 0;
691
+ while (initialLength < SYMLINK_BODY_PREFIX_BYTES.length) {
692
+ const next = await iterator.next();
693
+ if (next.done) break;
694
+ const chunk = bufferFromChunk(next.value);
695
+ if (chunk.length === 0) continue;
696
+ initialChunks.push(chunk);
697
+ initialLength += chunk.length;
698
+ }
570
699
 
571
700
  const dir = path.dirname(localPath);
572
701
  if (!fs.existsSync(dir)) {
@@ -592,18 +721,31 @@ export async function downloadFile(
592
721
  // The body is already buffered (tiny for symlink records); reading it here
593
722
  // is behavior-preserving for regular files (whose body never starts with
594
723
  // the prefix per the SYMLINK_BODY_PREFIX doc). See SYMLINK_BODY_PREFIX.
595
- const bodyString = objectBody.toString("utf-8");
724
+ const leading = leadingBytes(
725
+ initialChunks,
726
+ initialLength,
727
+ SYMLINK_BODY_PREFIX_BYTES.length,
728
+ );
729
+ const bodyHasSymlinkPrefix =
730
+ leading.length >= SYMLINK_BODY_PREFIX_BYTES.length &&
731
+ leading.subarray(0, SYMLINK_BODY_PREFIX_BYTES.length).equals(
732
+ SYMLINK_BODY_PREFIX_BYTES,
733
+ );
596
734
  const isSymlinkRecord =
597
735
  (typeof symlinkMarker === "string" && symlinkMarker.length > 0) ||
598
- bodyString.startsWith(SYMLINK_BODY_PREFIX);
736
+ bodyHasSymlinkPrefix;
599
737
  if (isSymlinkRecord) {
738
+ const objectBody = await collectRemainingChunks(initialChunks, iterator);
600
739
  // The target lives in the body (marker-only metadata convention).
601
740
  // Symlink record bodies are bounded by target length (typically
602
741
  // <300 bytes for relative paths, hard-capped by S3's 5 GB object
603
- // size); the transport already buffered it.
742
+ // size); only this branch buffers the body so regular files can stream.
604
743
  let symlinkTarget: string;
605
- if (bodyString.startsWith(SYMLINK_BODY_PREFIX)) {
606
- symlinkTarget = bodyString.slice(SYMLINK_BODY_PREFIX.length);
744
+ if (bodyHasSymlinkPrefix) {
745
+ symlinkTarget = objectBody.toString(
746
+ "utf-8",
747
+ SYMLINK_BODY_PREFIX_BYTES.length,
748
+ );
607
749
  } else {
608
750
  // Backward-compat fallback: a legacy upload from earlier in
609
751
  // this PR's lifetime stored the target in metadata (raw or
@@ -619,127 +761,100 @@ export async function downloadFile(
619
761
 
620
762
  if (symlinkTarget.length === 0) {
621
763
  throw new Error(
622
- `Symlink record for ${key} had no target (body: ${bodyString.length} bytes, marker: ${symlinkMarker})`,
764
+ `Symlink record for ${key} had no target (body: ${objectBody.length} bytes, marker: ${symlinkMarker})`,
623
765
  );
624
766
  }
625
767
 
626
- // Replace whatever's at localPath. unlink covers regular files; for
627
- // a stale symlink this also frees the slot. ENOENT is fine — first
628
- // download of this key has nothing to clear. Other errors propagate
629
- // because they signal a real problem (permissions, parent missing).
768
+ const tempPath = downloadTempPath(localPath);
630
769
  try {
631
- fs.unlinkSync(localPath);
632
- } catch (err: unknown) {
633
- if (
634
- !err ||
635
- typeof err !== "object" ||
636
- !("code" in err) ||
637
- (err as { code?: string }).code !== "ENOENT"
638
- ) {
639
- throw err;
640
- }
770
+ fs.symlinkSync(symlinkTarget, tempPath);
771
+ fs.renameSync(tempPath, localPath);
772
+ } catch (err) {
773
+ removeTempPath(tempPath);
774
+ throw err;
641
775
  }
642
- fs.symlinkSync(symlinkTarget, localPath);
643
776
  return { metadata };
644
777
  }
645
-
646
- // Symmetric to the symlink branch above: when a key was previously a
647
- // symlink and is later replaced in S3 by a regular object, the local
648
- // path still holds the stale symlink from the last sync. Without this
649
- // unlink, fs.writeFileSync follows the link and overwrites its
650
- // target file's contents — leaving the link in place and the new
651
- // regular object never materializing at the intended path. lstat
652
- // (not statSync) avoids following the link to test what's there.
778
+ const tempPath = downloadTempPath(localPath);
779
+ let tempReady = true;
780
+ let streamed: { hash: string; size: number };
653
781
  try {
654
- const existing = fs.lstatSync(localPath);
655
- if (existing.isSymbolicLink()) {
656
- fs.unlinkSync(localPath);
657
- }
658
- } catch (err: unknown) {
659
- // ENOENT means nothing's there; let writeFileSync handle creation.
660
- if (
661
- err &&
662
- typeof err === "object" &&
663
- "code" in err &&
664
- (err as { code?: string }).code !== "ENOENT"
665
- ) {
666
- throw err;
667
- }
668
- }
669
-
670
- fs.writeFileSync(localPath, objectBody);
671
-
672
- // Bug #5 apply source-side mode after the byte write. See
673
- // FILE_MODE_META_KEY for the metadata contract. Parses defensively:
674
- // a malformed value falls through with no chmod so the umask default
675
- // applies, matching the legacy back-compat path. fs.chmodSync
676
- // follows symlinks that's fine here because we're on the regular-
677
- // file branch (the symlink branch above already returned).
678
- //
679
- // Codex P2 (PR #24 round 3): strict octal-only regex BEFORE parseInt.
680
- // parseInt(modeOctal, 8) accepts partial-prefix garbage — "755junk"
681
- // parses to 0o755 instead of NaN — so tampered or malformed metadata
682
- // could still change local permissions unexpectedly. The regex
683
- // requires 1–4 pure octal digits (`[0-7]{1,4}$`), which matches what
684
- // the upload side stamps (`(mode & 0o777).toString(8)` → at most
685
- // three digits, all 0–7) and rejects everything else.
686
- const modeOctal = metadata?.[FILE_MODE_META_KEY];
687
- if (typeof modeOctal === "string" && /^[0-7]{1,4}$/.test(modeOctal)) {
688
- const parsed = parseInt(modeOctal, 8);
689
- if (Number.isFinite(parsed) && parsed >= 0 && parsed <= 0o777) {
690
- try {
691
- fs.chmodSync(localPath, parsed);
692
- } catch {
693
- // chmod failure (read-only FS, EPERM) is non-fatal — the file
694
- // is materialized, just with the umask default. Surface via
695
- // S3-side metadata being present but the file not matching;
696
- // a future operator-side audit can reconcile.
782
+ streamed = await streamRegularFileToTemp(tempPath, initialChunks, iterator);
783
+
784
+ // Bug #5 — apply source-side mode after the byte write. See
785
+ // FILE_MODE_META_KEY for the metadata contract. Parses defensively:
786
+ // a malformed value falls through with no chmod so the umask default
787
+ // applies, matching the legacy back-compat path. The staged path is a
788
+ // regular file, then it is atomically renamed over the destination.
789
+ //
790
+ // Codex P2 (PR #24 round 3): strict octal-only regex BEFORE parseInt.
791
+ // parseInt(modeOctal, 8) accepts partial-prefix garbage — "755junk"
792
+ // parses to 0o755 instead of NaN — so tampered or malformed metadata
793
+ // could still change local permissions unexpectedly. The regex
794
+ // requires 1–4 pure octal digits (`[0-7]{1,4}$`), which matches what
795
+ // the upload side stamps (`(mode & 0o777).toString(8)` → at most
796
+ // three digits, all 0–7) and rejects everything else.
797
+ const modeOctal = metadata?.[FILE_MODE_META_KEY];
798
+ if (typeof modeOctal === "string" && /^[0-7]{1,4}$/.test(modeOctal)) {
799
+ const parsed = parseInt(modeOctal, 8);
800
+ if (Number.isFinite(parsed) && parsed >= 0 && parsed <= 0o777) {
801
+ try {
802
+ fs.chmodSync(tempPath, parsed);
803
+ } catch {
804
+ // chmod failure (read-only FS, EPERM) is non-fatal the file
805
+ // is materialized, just with the umask default. Surface via
806
+ // S3-side metadata being present but the file not matching;
807
+ // a future operator-side audit can reconcile.
808
+ }
697
809
  }
698
810
  }
699
- }
700
811
 
701
- // 5.37.0 — apply source-side mtime after the byte write (and after the
702
- // chmod above; ordering between chmod and utimes doesn't matter, but
703
- // both must run AFTER writeFileSync because writeFileSync resets mtime
704
- // to wall-clock-now). See FILE_MTIME_META_KEY for the metadata contract.
705
- //
706
- // Strict-numeric regex BEFORE parseInt — same Codex P2 lesson as hq-mode.
707
- // `^-?[0-9]{1,16}$` rejects partial-prefix garbage ("175junk" → 175),
708
- // empty, double-signed, decimals, whitespace, and oversized strings. A
709
- // single optional leading `-` is allowed so pre-epoch / reproducible-build
710
- // timestamps round-trip (Codex PR #27 P2 — `mtimeMs === 0` and negative
711
- // epoch values are legitimate). 16 digits comfortably covers any plausible
712
- // epoch-ms value (year ~5138 is 16 digits; we'll cross that bridge later).
713
- //
714
- // fs.utimesSync FOLLOWS symlinks that's fine here because we're on
715
- // the regular-file branch (the symlink branch above already returned
716
- // before we reach this code).
717
- //
718
- // Composition with the 5.36.0 lstat fast-path: the journal stamp at the
719
- // share/sync call sites runs AFTER downloadFile returns, so the lstat
720
- // it captures sees the post-utimes mtime. Verified in cli/sync.ts
721
- // (downloadFile lstatSync updateEntry) and cli/share.ts (pull path
722
- // similarly lstats after downloadFile). If a future caller stamps the
723
- // journal BEFORE downloadFile completes, the fast-path will stale and
724
- // re-hash every sync forever keep the call-site invariant intact.
725
- const mtimeRaw = metadata?.[FILE_MTIME_META_KEY];
726
- if (typeof mtimeRaw === "string" && /^-?[0-9]{1,16}$/.test(mtimeRaw)) {
727
- const mtimeMs = parseInt(mtimeRaw, 10);
728
- if (Number.isFinite(mtimeMs)) {
729
- try {
730
- // utimesSync accepts seconds OR Date; use Date(ms) for precision.
731
- // atime = mtime is fine many filesystems are mounted noatime
732
- // and distinguishing "access" vs "modification" time doesn't
733
- // matter for sync semantics. Setting both keeps the on-disk
734
- // state deterministic across receivers.
735
- const mtimeDate = new Date(mtimeMs);
736
- fs.utimesSync(localPath, mtimeDate, mtimeDate);
737
- } catch {
738
- // EPERM / read-only FS / file just unlinked → non-fatal. The
739
- // file is materialized at write-time mtime; the source-of-truth
740
- // can re-sync next pass.
812
+ // 5.37.0 — apply source-side mtime after the byte write (and after the
813
+ // chmod above; ordering between chmod and utimes doesn't matter, but
814
+ // both must run AFTER writeFileSync because writeFileSync resets mtime
815
+ // to wall-clock-now). See FILE_MTIME_META_KEY for the metadata contract.
816
+ //
817
+ // Strict-numeric regex BEFORE parseInt — same Codex P2 lesson as hq-mode.
818
+ // `^-?[0-9]{1,16}$` rejects partial-prefix garbage ("175junk" → 175),
819
+ // empty, double-signed, decimals, whitespace, and oversized strings. A
820
+ // single optional leading `-` is allowed so pre-epoch / reproducible-build
821
+ // timestamps round-trip (Codex PR #27 P2 — `mtimeMs === 0` and negative
822
+ // epoch values are legitimate). 16 digits comfortably covers any plausible
823
+ // epoch-ms value (year ~5138 is 16 digits; we'll cross that bridge later).
824
+ //
825
+ // The staged file is regular; rename preserves these stamps at localPath.
826
+ //
827
+ // Composition with the 5.36.0 lstat fast-path: the journal stamp at the
828
+ // share/sync call sites runs AFTER downloadFile returns, so the lstat
829
+ // it captures sees the post-utimes mtime. Verified in cli/sync.ts
830
+ // (downloadFile lstatSync updateEntry) and cli/share.ts (pull path
831
+ // similarly lstats after downloadFile). If a future caller stamps the
832
+ // journal BEFORE downloadFile completes, the fast-path will stale and
833
+ // re-hash every sync forever keep the call-site invariant intact.
834
+ const mtimeRaw = metadata?.[FILE_MTIME_META_KEY];
835
+ if (typeof mtimeRaw === "string" && /^-?[0-9]{1,16}$/.test(mtimeRaw)) {
836
+ const mtimeMs = parseInt(mtimeRaw, 10);
837
+ if (Number.isFinite(mtimeMs)) {
838
+ try {
839
+ // utimesSync accepts seconds OR Date; use Date(ms) for precision.
840
+ // atime = mtime is fine — many filesystems are mounted noatime
841
+ // and distinguishing "access" vs "modification" time doesn't
842
+ // matter for sync semantics. Setting both keeps the on-disk
843
+ // state deterministic across receivers.
844
+ const mtimeDate = new Date(mtimeMs);
845
+ fs.utimesSync(tempPath, mtimeDate, mtimeDate);
846
+ } catch {
847
+ // EPERM / read-only FS / file just unlinked → non-fatal. The
848
+ // file is materialized at write-time mtime; the source-of-truth
849
+ // can re-sync next pass.
850
+ }
741
851
  }
742
852
  }
853
+
854
+ fs.renameSync(tempPath, localPath);
855
+ tempReady = false;
856
+ } finally {
857
+ if (tempReady) removeTempPath(tempPath);
743
858
  }
744
859
 
745
860
  // TODO: stamp hq-btime once Node lands lbirthtime (or birthtimeSync).
@@ -747,7 +862,7 @@ export async function downloadFile(
747
862
  // distinct creation time, so a future receiver upgrade picks it up
748
863
  // automatically without a server-side data migration.
749
864
 
750
- return { metadata };
865
+ return { metadata, contentHash: streamed.hash, contentSize: streamed.size };
751
866
  }
752
867
 
753
868
  export interface RemoteFile {
@@ -28,7 +28,10 @@ import type {
28
28
  SyncJournal,
29
29
  } from "./types.js";
30
30
  import { hashFile, tombstoneEntry } from "./journal.js";
31
- import { isCoveredByAny } from "./prefix-coalesce.js";
31
+ import {
32
+ isCoveredByAny,
33
+ type ScopePrefixInput,
34
+ } from "./prefix-coalesce.js";
32
35
 
33
36
  export interface OrphanClassification {
34
37
  /** Relative path (journal key). */
@@ -59,9 +62,9 @@ export interface BuildScopeShrinkPlanInput {
59
62
  journal: SyncJournal;
60
63
  hqRoot: string;
61
64
  /** Coalesced prefixes used by the LAST pull for this company. */
62
- lastPrefixSet: string[];
65
+ lastPrefixSet: readonly ScopePrefixInput[];
63
66
  /** Coalesced prefixes the CURRENT pull will use. */
64
- currentPrefixSet: string[];
67
+ currentPrefixSet: readonly ScopePrefixInput[];
65
68
  /**
66
69
  * The caller's own Cognito `sub`. When set, a file the caller authored
67
70
  * (`entry.createdBySub === callerSub`) is NEVER orphaned by a scope shrink —
@@ -536,6 +536,115 @@ describe("collectAndSendSkillTelemetry — hqRoot scoping", () => {
536
536
  await fs.rm(tmp, { recursive: true, force: true });
537
537
  });
538
538
 
539
+ it("R-F19: bounds a single oversized sanitized skill row before POST", async () => {
540
+ const tmp = await fs.mkdtemp(path.join(os.tmpdir(), "skill-tel-r-f19-"));
541
+ const projects = path.join(tmp, "projects");
542
+ const dir = path.join(projects, "-p");
543
+ await fs.mkdir(dir, { recursive: true });
544
+ const hugeSkill = `deploy-${"x".repeat(400_000)}`;
545
+ await fs.writeFile(
546
+ path.join(dir, "s.jsonl"),
547
+ row({
548
+ type: "user",
549
+ sessionId: "s-r-f19-singleton",
550
+ timestamp: "2026-06-19T11:00:00.000Z",
551
+ cwd: "/x",
552
+ uuid: "u-r-f19-singleton",
553
+ message: {
554
+ role: "user",
555
+ content: `<command-name>/${hugeSkill}</command-name>`,
556
+ },
557
+ }) + "\n",
558
+ "utf-8",
559
+ );
560
+
561
+ const captured: SkillInvocationBatch[] = [];
562
+ const logs: string[] = [];
563
+ const result = await collectAndSendSkillTelemetry({
564
+ client: stubClient(captured),
565
+ machineId: "m",
566
+ installerVersion: "t",
567
+ claudeProjectsRoot: projects,
568
+ codexSessionsRoot: path.join(tmp, "codex"),
569
+ cursorPath: path.join(tmp, "cursor.json"),
570
+ log: (msg) => logs.push(msg),
571
+ });
572
+
573
+ expect(result.eventsSent).toBe(1);
574
+ expect(captured).toHaveLength(1);
575
+ const wireSize = Buffer.byteLength(JSON.stringify(captured[0]), "utf-8");
576
+ expect(wireSize).toBeLessThanOrEqual(240 * 1024);
577
+ expect(captured[0].events[0].skill).not.toBe(hugeSkill);
578
+ expect(logs.some((line) => line.includes("oversized row truncated"))).toBe(
579
+ true,
580
+ );
581
+
582
+ await fs.rm(tmp, { recursive: true, force: true });
583
+ });
584
+
585
+ it("R-F19: a failed earlier skill batch prevents cursor advancement past it", async () => {
586
+ const tmp = await fs.mkdtemp(path.join(os.tmpdir(), "skill-tel-r-f19-"));
587
+ const projects = path.join(tmp, "projects");
588
+ const dir = path.join(projects, "-p");
589
+ await fs.mkdir(dir, { recursive: true });
590
+ const file = path.join(dir, "s.jsonl");
591
+ await fs.writeFile(
592
+ file,
593
+ Array.from({ length: 101 }, (_, i) =>
594
+ row({
595
+ type: "user",
596
+ sessionId: "s-r-f19-prefix",
597
+ timestamp: `2026-06-19T12:${String(Math.floor(i / 60)).padStart(2, "0")}:${String(i % 60).padStart(2, "0")}.000Z`,
598
+ cwd: "/x",
599
+ uuid: `u-r-f19-${i}`,
600
+ message: {
601
+ role: "user",
602
+ content: `<command-name>/skill-${i}</command-name>`,
603
+ },
604
+ }),
605
+ ).join("\n") + "\n",
606
+ "utf-8",
607
+ );
608
+ const cursorPath = path.join(tmp, "cursor.json");
609
+ const captured: SkillInvocationBatch[] = [];
610
+ let calls = 0;
611
+ const client = {
612
+ async getTelemetryOptIn() {
613
+ return { enabled: true, updatedAt: null };
614
+ },
615
+ async postSkillInvocations(batch: SkillInvocationBatch) {
616
+ calls++;
617
+ if (calls === 1) throw new Error("first batch failed");
618
+ captured.push(batch);
619
+ return { ok: true, written: batch.events.length, skipped: [] };
620
+ },
621
+ };
622
+ const opts = {
623
+ client,
624
+ machineId: "m",
625
+ installerVersion: "t",
626
+ claudeProjectsRoot: projects,
627
+ codexSessionsRoot: path.join(tmp, "codex"),
628
+ cursorPath,
629
+ };
630
+
631
+ const first = await collectAndSendSkillTelemetry(opts);
632
+ expect(first.eventsSent).toBe(0);
633
+ expect(calls).toBe(1);
634
+ const cursorAfterFailure = JSON.parse(
635
+ await fs.readFile(cursorPath, "utf-8"),
636
+ ) as { files: Record<string, { offset: number }> };
637
+ expect(cursorAfterFailure.files[file]?.offset ?? 0).toBe(0);
638
+
639
+ const second = await collectAndSendSkillTelemetry(opts);
640
+ expect(second.eventsSent).toBe(101);
641
+ expect(captured.flatMap((b) => b.events.map((e) => e.skill))).toContain(
642
+ "skill-0",
643
+ );
644
+
645
+ await fs.rm(tmp, { recursive: true, force: true });
646
+ });
647
+
539
648
  it("picks up only newly-appended events on a later run (incremental offset)", async () => {
540
649
  const tmp = await fs.mkdtemp(path.join(os.tmpdir(), "skill-tel-"));
541
650
  const projects = path.join(tmp, "projects");