@indigoai-us/hq-cloud 6.11.10 → 6.11.12

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (173) hide show
  1. package/dist/bin/sync-runner.d.ts +2 -0
  2. package/dist/bin/sync-runner.d.ts.map +1 -1
  3. package/dist/bin/sync-runner.js +231 -52
  4. package/dist/bin/sync-runner.js.map +1 -1
  5. package/dist/bin/sync-runner.test.js +330 -11
  6. package/dist/bin/sync-runner.test.js.map +1 -1
  7. package/dist/cli/reindex.d.ts.map +1 -1
  8. package/dist/cli/reindex.js +16 -1
  9. package/dist/cli/reindex.js.map +1 -1
  10. package/dist/cli/reindex.test.js +39 -1
  11. package/dist/cli/reindex.test.js.map +1 -1
  12. package/dist/cli/rescue-classify-ordering.test.js +58 -0
  13. package/dist/cli/rescue-classify-ordering.test.js.map +1 -1
  14. package/dist/cli/rescue-core.js +229 -15
  15. package/dist/cli/rescue-core.js.map +1 -1
  16. package/dist/cli/rescue-exec-bit-preserve.test.d.ts +2 -0
  17. package/dist/cli/rescue-exec-bit-preserve.test.d.ts.map +1 -0
  18. package/dist/cli/rescue-exec-bit-preserve.test.js +169 -0
  19. package/dist/cli/rescue-exec-bit-preserve.test.js.map +1 -0
  20. package/dist/cli/share.d.ts +2 -1
  21. package/dist/cli/share.d.ts.map +1 -1
  22. package/dist/cli/share.js +100 -32
  23. package/dist/cli/share.js.map +1 -1
  24. package/dist/cli/share.test.js +30 -0
  25. package/dist/cli/share.test.js.map +1 -1
  26. package/dist/cli/sync.d.ts +28 -1
  27. package/dist/cli/sync.d.ts.map +1 -1
  28. package/dist/cli/sync.js +188 -59
  29. package/dist/cli/sync.js.map +1 -1
  30. package/dist/cli/sync.test.js +487 -1
  31. package/dist/cli/sync.test.js.map +1 -1
  32. package/dist/cognito-auth.d.ts.map +1 -1
  33. package/dist/cognito-auth.js +55 -10
  34. package/dist/cognito-auth.js.map +1 -1
  35. package/dist/cognito-auth.test.js +61 -0
  36. package/dist/cognito-auth.test.js.map +1 -1
  37. package/dist/index.d.ts +2 -1
  38. package/dist/index.d.ts.map +1 -1
  39. package/dist/index.js +1 -1
  40. package/dist/index.js.map +1 -1
  41. package/dist/journal.d.ts.map +1 -1
  42. package/dist/journal.js +93 -6
  43. package/dist/journal.js.map +1 -1
  44. package/dist/journal.test.js +59 -0
  45. package/dist/journal.test.js.map +1 -1
  46. package/dist/machine-auth.test.js +60 -2
  47. package/dist/machine-auth.test.js.map +1 -1
  48. package/dist/object-io.d.ts +37 -1
  49. package/dist/object-io.d.ts.map +1 -1
  50. package/dist/object-io.js +148 -29
  51. package/dist/object-io.js.map +1 -1
  52. package/dist/object-io.test.js +121 -0
  53. package/dist/object-io.test.js.map +1 -1
  54. package/dist/operation-lock.d.ts +8 -8
  55. package/dist/operation-lock.d.ts.map +1 -1
  56. package/dist/operation-lock.js +99 -32
  57. package/dist/operation-lock.js.map +1 -1
  58. package/dist/operation-lock.test.js +51 -4
  59. package/dist/operation-lock.test.js.map +1 -1
  60. package/dist/personal-vault.d.ts +8 -0
  61. package/dist/personal-vault.d.ts.map +1 -1
  62. package/dist/personal-vault.js +17 -3
  63. package/dist/personal-vault.js.map +1 -1
  64. package/dist/personal-vault.test.js +34 -0
  65. package/dist/personal-vault.test.js.map +1 -1
  66. package/dist/prefix-coalesce.d.ts +20 -9
  67. package/dist/prefix-coalesce.d.ts.map +1 -1
  68. package/dist/prefix-coalesce.js +124 -28
  69. package/dist/prefix-coalesce.js.map +1 -1
  70. package/dist/prefix-coalesce.test.js +57 -2
  71. package/dist/prefix-coalesce.test.js.map +1 -1
  72. package/dist/remote-pull.d.ts +6 -1
  73. package/dist/remote-pull.d.ts.map +1 -1
  74. package/dist/remote-pull.js +62 -13
  75. package/dist/remote-pull.js.map +1 -1
  76. package/dist/remote-pull.test.js +189 -0
  77. package/dist/remote-pull.test.js.map +1 -1
  78. package/dist/s3.d.ts +2 -0
  79. package/dist/s3.d.ts.map +1 -1
  80. package/dist/s3.js +197 -116
  81. package/dist/s3.js.map +1 -1
  82. package/dist/s3.test.js +109 -0
  83. package/dist/s3.test.js.map +1 -1
  84. package/dist/scope-shrink.d.ts +3 -2
  85. package/dist/scope-shrink.d.ts.map +1 -1
  86. package/dist/scope-shrink.js +1 -1
  87. package/dist/scope-shrink.js.map +1 -1
  88. package/dist/skill-telemetry.d.ts +1 -1
  89. package/dist/skill-telemetry.d.ts.map +1 -1
  90. package/dist/skill-telemetry.js +69 -9
  91. package/dist/skill-telemetry.js.map +1 -1
  92. package/dist/skill-telemetry.test.js +86 -0
  93. package/dist/skill-telemetry.test.js.map +1 -1
  94. package/dist/sync/event-sync.d.ts +6 -0
  95. package/dist/sync/event-sync.d.ts.map +1 -1
  96. package/dist/sync/event-sync.js +34 -1
  97. package/dist/sync/event-sync.js.map +1 -1
  98. package/dist/sync/event-sync.test.js +73 -0
  99. package/dist/sync/event-sync.test.js.map +1 -1
  100. package/dist/sync/metrics.d.ts +17 -1
  101. package/dist/sync/metrics.d.ts.map +1 -1
  102. package/dist/sync/metrics.js +32 -1
  103. package/dist/sync/metrics.js.map +1 -1
  104. package/dist/sync/metrics.test.js +74 -1
  105. package/dist/sync/metrics.test.js.map +1 -1
  106. package/dist/sync/pull-scope.d.ts.map +1 -1
  107. package/dist/sync/pull-scope.js +15 -7
  108. package/dist/sync/pull-scope.js.map +1 -1
  109. package/dist/sync/push-receiver.d.ts +6 -5
  110. package/dist/sync/push-receiver.d.ts.map +1 -1
  111. package/dist/sync/push-receiver.js +13 -15
  112. package/dist/sync/push-receiver.js.map +1 -1
  113. package/dist/sync/push-receiver.test.js +36 -1
  114. package/dist/sync/push-receiver.test.js.map +1 -1
  115. package/dist/telemetry.d.ts +1 -1
  116. package/dist/telemetry.d.ts.map +1 -1
  117. package/dist/telemetry.js +59 -6
  118. package/dist/telemetry.js.map +1 -1
  119. package/dist/telemetry.test.js +74 -0
  120. package/dist/telemetry.test.js.map +1 -1
  121. package/dist/types.d.ts +8 -0
  122. package/dist/types.d.ts.map +1 -1
  123. package/dist/watcher.d.ts +36 -0
  124. package/dist/watcher.d.ts.map +1 -1
  125. package/dist/watcher.js +152 -30
  126. package/dist/watcher.js.map +1 -1
  127. package/dist/watcher.test.js +103 -0
  128. package/dist/watcher.test.js.map +1 -1
  129. package/package.json +1 -1
  130. package/src/bin/sync-runner.test.ts +396 -11
  131. package/src/bin/sync-runner.ts +254 -52
  132. package/src/cli/reindex.test.ts +47 -1
  133. package/src/cli/reindex.ts +17 -1
  134. package/src/cli/rescue-classify-ordering.test.ts +61 -0
  135. package/src/cli/rescue-core.ts +261 -15
  136. package/src/cli/rescue-exec-bit-preserve.test.ts +187 -0
  137. package/src/cli/share.test.ts +38 -0
  138. package/src/cli/share.ts +103 -34
  139. package/src/cli/sync.test.ts +594 -1
  140. package/src/cli/sync.ts +229 -65
  141. package/src/cognito-auth.test.ts +77 -0
  142. package/src/cognito-auth.ts +73 -11
  143. package/src/index.ts +8 -0
  144. package/src/journal.test.ts +72 -0
  145. package/src/journal.ts +95 -8
  146. package/src/machine-auth.test.ts +64 -2
  147. package/src/object-io.test.ts +142 -0
  148. package/src/object-io.ts +182 -30
  149. package/src/operation-lock.test.ts +63 -4
  150. package/src/operation-lock.ts +99 -31
  151. package/src/personal-vault.test.ts +42 -0
  152. package/src/personal-vault.ts +18 -3
  153. package/src/prefix-coalesce.test.ts +71 -1
  154. package/src/prefix-coalesce.ts +155 -30
  155. package/src/remote-pull.test.ts +205 -0
  156. package/src/remote-pull.ts +77 -14
  157. package/src/s3.test.ts +126 -0
  158. package/src/s3.ts +237 -122
  159. package/src/scope-shrink.ts +6 -3
  160. package/src/skill-telemetry.test.ts +109 -0
  161. package/src/skill-telemetry.ts +82 -14
  162. package/src/sync/event-sync.test.ts +75 -0
  163. package/src/sync/event-sync.ts +54 -1
  164. package/src/sync/metrics.test.ts +81 -0
  165. package/src/sync/metrics.ts +59 -4
  166. package/src/sync/pull-scope.ts +23 -7
  167. package/src/sync/push-receiver.test.ts +38 -1
  168. package/src/sync/push-receiver.ts +15 -18
  169. package/src/telemetry.test.ts +85 -0
  170. package/src/telemetry.ts +69 -6
  171. package/src/types.ts +8 -0
  172. package/src/watcher.test.ts +117 -0
  173. package/src/watcher.ts +209 -33
package/src/object-io.ts CHANGED
@@ -104,6 +104,12 @@ export interface GetObjectResult {
104
104
  metadata?: Record<string, string>;
105
105
  }
106
106
 
107
+ export interface GetObjectStreamResult {
108
+ body: AsyncIterable<Uint8Array>;
109
+ /** S3 user metadata (keys lowercased by S3). */
110
+ metadata?: Record<string, string>;
111
+ }
112
+
107
113
  export interface ListObjectsInput {
108
114
  prefix?: string;
109
115
  continuationToken?: string;
@@ -137,6 +143,7 @@ export interface HeadObjectResult {
137
143
  export interface ObjectIO {
138
144
  putObject(input: PutObjectInput): Promise<{ etag: string }>;
139
145
  getObject(key: string): Promise<GetObjectResult>;
146
+ getObjectStream?(key: string): Promise<GetObjectStreamResult>;
140
147
  listObjects(input: ListObjectsInput): Promise<ListObjectsResult>;
141
148
  deleteObject(key: string): Promise<void>;
142
149
  /**
@@ -243,15 +250,23 @@ export class S3SdkObjectIO implements ObjectIO {
243
250
  return { etag: res.ETag || "" };
244
251
  }
245
252
 
246
- async getObject(key: string): Promise<GetObjectResult> {
253
+ async getObjectStream(key: string): Promise<GetObjectStreamResult> {
247
254
  const res = await this.client.send(
248
255
  new GetObjectCommand({ Bucket: this.bucket, Key: key }),
249
256
  );
250
257
  if (!res.Body) {
251
258
  throw new Error(`Empty response for ${key}`);
252
259
  }
253
- const body = await drainToBuffer(res.Body as AsyncIterable<Uint8Array>);
254
- return { body, metadata: res.Metadata };
260
+ return {
261
+ body: res.Body as AsyncIterable<Uint8Array>,
262
+ metadata: res.Metadata,
263
+ };
264
+ }
265
+
266
+ async getObject(key: string): Promise<GetObjectResult> {
267
+ const res = await this.getObjectStream(key);
268
+ const body = await drainToBuffer(res.body);
269
+ return { body, metadata: res.metadata };
255
270
  }
256
271
 
257
272
  async listObjects(input: ListObjectsInput): Promise<ListObjectsResult> {
@@ -416,6 +431,19 @@ const PRIME_CONCURRENCY = 4;
416
431
  /** Treat a cached URL within this window of expiry as a miss (re-presign). */
417
432
  const CACHE_SAFETY_MS = 60_000;
418
433
 
434
+ /**
435
+ * Opt-in gate for strict fail-closed enforcement of fenced presigned PUTs.
436
+ *
437
+ * Defaults OFF until the hq-pro files-presign server signs and echoes
438
+ * If-Match/If-None-Match. While disabled, the presigned transport preserves
439
+ * today's behavior: replay whatever signed headers the server returned.
440
+ */
441
+ export const PRESIGN_FENCE_STRICT_ENV_VAR = "HQ_PRESIGN_FENCE_STRICT";
442
+
443
+ function isPresignFenceStrictEnabled(): boolean {
444
+ return process.env[PRESIGN_FENCE_STRICT_ENV_VAR] === "true";
445
+ }
446
+
419
447
  interface CacheEntry {
420
448
  url: string;
421
449
  headers?: Record<string, string>;
@@ -441,6 +469,24 @@ export class RateLimitedError extends Error {
441
469
  }
442
470
  }
443
471
 
472
+ /**
473
+ * A fenced presigned PUT is only safe if the presign service signed and echoed
474
+ * the requested conditional header for replay. Missing/mismatched condition
475
+ * headers mean an older server or a stale primed URL would write
476
+ * unconditionally, so fail closed and let the caller retry later.
477
+ */
478
+ export class PresignPreconditionMissingError extends Error {
479
+ readonly retryable = true;
480
+
481
+ constructor(
482
+ readonly key: string,
483
+ readonly header: "if-match" | "if-none-match",
484
+ ) {
485
+ super(`presigned PUT for ${key} missing required ${header} precondition`);
486
+ this.name = "PresignPreconditionMissing";
487
+ }
488
+ }
489
+
444
490
  /**
445
491
  * One-way circuit breaker shared across a run's per-company transports. The
446
492
  * first 429 (vault rate budget exhausted) trips it; thereafter every UNCACHED
@@ -483,6 +529,7 @@ function isRateLimit(err: unknown): boolean {
483
529
  */
484
530
  export class PresignObjectIO implements ObjectIO {
485
531
  private readonly urlCache = new Map<string, CacheEntry>();
532
+ private readonly headReuseCache = new Map<string, CacheEntry>();
486
533
 
487
534
  constructor(
488
535
  private readonly vault: PresignTransportClient,
@@ -497,26 +544,45 @@ export class PresignObjectIO implements ObjectIO {
497
544
  }
498
545
 
499
546
  hasPrimedPut(key: string): boolean {
500
- return this.cached("put", key) !== undefined;
547
+ return this.peekCached("put", key) !== undefined;
501
548
  }
502
549
 
503
550
  /** A live (non-expiring) cached URL for op+key, or undefined. */
504
- private cached(op: PresignOp, key: string): CacheEntry | undefined {
505
- const hit = this.urlCache.get(this.cacheKey(op, key));
551
+ private peekCached(op: PresignOp, key: string): CacheEntry | undefined {
552
+ const cacheKey = this.cacheKey(op, key);
553
+ const hit = this.urlCache.get(cacheKey);
506
554
  if (!hit) return undefined;
507
555
  if (Date.now() >= hit.expiresAtMs - CACHE_SAFETY_MS) {
508
- this.urlCache.delete(this.cacheKey(op, key));
556
+ this.urlCache.delete(cacheKey);
509
557
  return undefined;
510
558
  }
511
559
  return hit;
512
560
  }
513
561
 
514
- /**
515
- * Resolve a presigned URL (+ replay headers) for op+key: cache hit if primed,
516
- * else a single presign. Throws on per-key denial (matches the SDK path's
517
- * access error). `extra` carries PUT contentType/metadata on the miss path.
518
- */
519
- private async resolveUrl(
562
+ /** Return and evict a live cached URL so primed batches are bounded. */
563
+ private consumeCached(op: PresignOp, key: string): CacheEntry | undefined {
564
+ const hit = this.peekCached(op, key);
565
+ if (hit) this.urlCache.delete(this.cacheKey(op, key));
566
+ return hit;
567
+ }
568
+
569
+ private peekHeadReuse(key: string): CacheEntry | undefined {
570
+ const hit = this.headReuseCache.get(key);
571
+ if (!hit) return undefined;
572
+ if (Date.now() >= hit.expiresAtMs - CACHE_SAFETY_MS) {
573
+ this.headReuseCache.delete(key);
574
+ return undefined;
575
+ }
576
+ return hit;
577
+ }
578
+
579
+ private consumeHeadReuse(key: string): CacheEntry | undefined {
580
+ const hit = this.peekHeadReuse(key);
581
+ if (hit) this.headReuseCache.delete(key);
582
+ return hit;
583
+ }
584
+
585
+ private async presignSingle(
520
586
  op: PresignOp,
521
587
  key: string,
522
588
  extra?: {
@@ -527,10 +593,6 @@ export class PresignObjectIO implements ObjectIO {
527
593
  ifNoneMatch?: "*";
528
594
  },
529
595
  ): Promise<{ url: string; headers?: Record<string, string> }> {
530
- const hit = this.cached(op, key);
531
- if (hit) return { url: hit.url, headers: hit.headers };
532
- // Primed URLs still serve once the breaker trips (no wire needed); only an
533
- // uncached key on an exhausted budget fails fast.
534
596
  if (this.breaker.isTripped()) throw new RateLimitedError(key, op);
535
597
  let results;
536
598
  try {
@@ -550,6 +612,71 @@ export class PresignObjectIO implements ObjectIO {
550
612
  return { url: row.url!, headers: row.headers };
551
613
  }
552
614
 
615
+ private async resolveGetUrlForBody(
616
+ key: string,
617
+ ): Promise<{ url: string; headers?: Record<string, string> }> {
618
+ const hit = this.consumeCached("get", key);
619
+ if (hit) {
620
+ this.headReuseCache.set(key, hit);
621
+ return { url: hit.url, headers: hit.headers };
622
+ }
623
+ this.headReuseCache.delete(key);
624
+ return this.presignSingle("get", key);
625
+ }
626
+
627
+ /**
628
+ * Resolve a presigned URL (+ replay headers) for op+key: cache hit if primed,
629
+ * else a single presign. Throws on per-key denial (matches the SDK path's
630
+ * access error). `extra` carries PUT contentType/metadata on the miss path.
631
+ */
632
+ private async resolveUrl(
633
+ op: PresignOp,
634
+ key: string,
635
+ extra?: {
636
+ contentType?: string;
637
+ metadata?: Record<string, string>;
638
+ /** Conditional-write fence for PUT presigns — see PutPrecondition. */
639
+ ifMatch?: string;
640
+ ifNoneMatch?: "*";
641
+ },
642
+ ): Promise<{ url: string; headers?: Record<string, string> }> {
643
+ const hasPutFence =
644
+ op === "put" &&
645
+ (extra?.ifMatch !== undefined || extra?.ifNoneMatch !== undefined);
646
+ if (hasPutFence) {
647
+ this.urlCache.delete(this.cacheKey("put", key));
648
+ }
649
+ const hit = hasPutFence ? undefined : this.consumeCached(op, key);
650
+ if (hit) return { url: hit.url, headers: hit.headers };
651
+ // Primed URLs still serve once the breaker trips (no wire needed); only an
652
+ // uncached key on an exhausted budget fails fast.
653
+ return this.presignSingle(op, key, extra);
654
+ }
655
+
656
+ private requireSignedPutPreconditions(
657
+ key: string,
658
+ input: PutPrecondition,
659
+ headers: Record<string, string> | undefined,
660
+ ): void {
661
+ const lowerHeaders = new Map<string, string>();
662
+ for (const [name, value] of Object.entries(headers ?? {})) {
663
+ lowerHeaders.set(name.toLowerCase(), value);
664
+ }
665
+
666
+ if (input.ifMatch) {
667
+ const signed = lowerHeaders.get("if-match");
668
+ if (signed !== quoteEtag(input.ifMatch)) {
669
+ throw new PresignPreconditionMissingError(key, "if-match");
670
+ }
671
+ }
672
+ if (input.ifNoneMatch) {
673
+ const signed = lowerHeaders.get("if-none-match");
674
+ if (signed !== input.ifNoneMatch) {
675
+ throw new PresignPreconditionMissingError(key, "if-none-match");
676
+ }
677
+ }
678
+ }
679
+
553
680
  async prime(op: PresignOp, keys: PresignKeyInput[]): Promise<void> {
554
681
  if (keys.length === 0) return;
555
682
  const chunks: PresignKeyInput[][] = [];
@@ -601,14 +728,10 @@ export class PresignObjectIO implements ObjectIO {
601
728
  async putObject(input: PutObjectInput): Promise<{ etag: string }> {
602
729
  // Conditional-write fields (ifMatch/ifNoneMatch) are forwarded on the
603
730
  // presign request so the server can sign If-Match/If-None-Match into the
604
- // URL and echo them via `headers` for replay. Until hq-pro's
605
- // files-presign signs them (follow-up to this PR), the server ignores
606
- // the fields and the returned header set carries no condition — the PUT
607
- // stays unconditional on this transport, exactly today's behavior. We
608
- // deliberately do NOT inject the header client-side: an unsigned
609
- // conditional header breaks the SigV4 signature. Enforcement on this
610
- // transport activates the moment the server starts signing; the SDK
611
- // transport enforces immediately.
731
+ // URL and echo them via `headers` for replay. Do not inject a missing
732
+ // condition client-side: an unsigned conditional header breaks the SigV4
733
+ // signature. Strict validation of echoed headers remains feature-gated
734
+ // until the hq-pro files-presign server signs these conditions.
612
735
  const row = await this.resolveUrl("put", input.key, {
613
736
  contentType: input.contentType,
614
737
  ...(input.metadata && Object.keys(input.metadata).length > 0
@@ -617,6 +740,9 @@ export class PresignObjectIO implements ObjectIO {
617
740
  ...(input.ifMatch ? { ifMatch: stripQuotes(input.ifMatch) } : {}),
618
741
  ...(input.ifNoneMatch ? { ifNoneMatch: input.ifNoneMatch } : {}),
619
742
  });
743
+ if (isPresignFenceStrictEnabled()) {
744
+ this.requireSignedPutPreconditions(input.key, input, row.headers);
745
+ }
620
746
  // The server signs Content-Type, SSE-KMS, and every x-amz-meta-* into the
621
747
  // signature and returns them in `headers`; they MUST be replayed verbatim
622
748
  // or SigV4 rejects the PUT.
@@ -642,8 +768,8 @@ export class PresignObjectIO implements ObjectIO {
642
768
  return { etag: stripQuotes(res.headers.get("etag") ?? undefined) };
643
769
  }
644
770
 
645
- async getObject(key: string): Promise<GetObjectResult> {
646
- const row = await this.resolveUrl("get", key);
771
+ async getObjectStream(key: string): Promise<GetObjectStreamResult> {
772
+ const row = await this.resolveGetUrlForBody(key);
647
773
  const res = await fetchWithRetry(row.url, { method: "GET" }, `presigned GET ${key}`);
648
774
  if (res.status === 404) {
649
775
  await cancelBody(res);
@@ -653,8 +779,19 @@ export class PresignObjectIO implements ObjectIO {
653
779
  const detail = await safeText(res);
654
780
  throw new Error(`presigned GET failed for ${key}: ${res.status} ${detail}`);
655
781
  }
656
- const body = Buffer.from(await res.arrayBuffer());
657
- return { body, metadata: metaFromHeaders(res.headers) };
782
+ if (!res.body) {
783
+ return { body: (async function* () {})(), metadata: metaFromHeaders(res.headers) };
784
+ }
785
+ return {
786
+ body: webReadableToAsyncIterable(res.body),
787
+ metadata: metaFromHeaders(res.headers),
788
+ };
789
+ }
790
+
791
+ async getObject(key: string): Promise<GetObjectResult> {
792
+ const res = await this.getObjectStream(key);
793
+ const body = await drainToBuffer(res.body);
794
+ return { body, metadata: res.metadata };
658
795
  }
659
796
 
660
797
  async listObjects(input: ListObjectsInput): Promise<ListObjectsResult> {
@@ -696,7 +833,7 @@ export class PresignObjectIO implements ObjectIO {
696
833
  // and conflict-detection call sites that use headObject. Reuses the GET
697
834
  // cache: a prime("get", …) before a pull warms these HEADs for free.
698
835
  let url: string;
699
- const hit = this.cached("get", key);
836
+ const hit = this.consumeHeadReuse(key) ?? this.consumeCached("get", key);
700
837
  if (hit) {
701
838
  url = hit.url;
702
839
  } else {
@@ -776,6 +913,21 @@ async function cancelBody(res: Response): Promise<void> {
776
913
  }
777
914
  }
778
915
 
916
+ async function* webReadableToAsyncIterable(
917
+ stream: ReadableStream<Uint8Array>,
918
+ ): AsyncIterable<Uint8Array> {
919
+ const reader = stream.getReader();
920
+ try {
921
+ while (true) {
922
+ const { done, value } = await reader.read();
923
+ if (done) return;
924
+ if (value && value.byteLength > 0) yield value;
925
+ }
926
+ } finally {
927
+ reader.releaseLock();
928
+ }
929
+ }
930
+
779
931
  function parseLastModified(value: string | null): Date {
780
932
  if (!value) return new Date();
781
933
  const d = new Date(value);
@@ -70,6 +70,27 @@ describe("operation-lock", () => {
70
70
  expect(lockPathFor(rootA + path.sep)).toBe(a);
71
71
  });
72
72
 
73
+ it("F14: symlink aliases share the same root lock", () => {
74
+ const aliasParent = fs.mkdtempSync(path.join(os.tmpdir(), "hq-rootA-link-"));
75
+ const aliasRoot = path.join(aliasParent, "alias");
76
+ fs.symlinkSync(rootA, aliasRoot, "dir");
77
+
78
+ let realHandle: ReturnType<typeof acquireOperationLock> | undefined;
79
+ let aliasHandle: ReturnType<typeof acquireOperationLock> | undefined;
80
+ try {
81
+ expect(lockPathFor(aliasRoot)).toBe(lockPathFor(rootA));
82
+
83
+ realHandle = acquireOperationLock(rootA, "sync");
84
+ expect(() => {
85
+ aliasHandle = acquireOperationLock(aliasRoot, "rescue", { wait: false });
86
+ }).toThrowError(OperationLockedError);
87
+ } finally {
88
+ aliasHandle?.release();
89
+ realHandle?.release();
90
+ fs.rmSync(aliasParent, { recursive: true, force: true });
91
+ }
92
+ });
93
+
73
94
  it("acquires, writes holder info, and releases (file gone after release)", () => {
74
95
  const h = acquireOperationLock(rootA, "sync");
75
96
  expect(fs.existsSync(h.path)).toBe(true);
@@ -224,13 +245,51 @@ describe("operation-lock", () => {
224
245
  h.release();
225
246
  });
226
247
 
227
- it("reclaims a torn/unreadable lock file", () => {
248
+ it("treats a torn/unreadable lock file as busy", () => {
228
249
  const p = lockPathFor(rootA);
229
250
  fs.mkdirSync(path.dirname(p), { recursive: true });
230
251
  fs.writeFileSync(p, "{ this is not valid json");
231
- const h = acquireOperationLock(rootA, "reindex");
232
- expect(fs.existsSync(h.path)).toBe(true);
233
- h.release();
252
+ expect(() => acquireOperationLock(rootA, "reindex", { wait: false })).toThrowError(
253
+ OperationLockedError,
254
+ );
255
+ expect(fs.readFileSync(p, "utf8")).toBe("{ this is not valid json");
256
+ });
257
+
258
+ it("F34: fresh empty lock files are not reclaimed as stale", () => {
259
+ const p = lockPathFor(rootA);
260
+ fs.mkdirSync(path.dirname(p), { recursive: true });
261
+ fs.closeSync(fs.openSync(p, "wx"));
262
+
263
+ let handle: ReturnType<typeof acquireOperationLock> | undefined;
264
+ try {
265
+ expect(() => {
266
+ handle = acquireOperationLock(rootA, "rescue", { wait: false });
267
+ }).toThrow();
268
+ expect(fs.existsSync(p)).toBe(true);
269
+ expect(fs.readFileSync(p, "utf8")).toBe("");
270
+ } finally {
271
+ handle?.release();
272
+ fs.rmSync(p, { force: true });
273
+ }
274
+ });
275
+
276
+ it("R-F34: initializing empty and torn locks are busy, not reclaimed", () => {
277
+ const p = lockPathFor(rootA);
278
+ fs.mkdirSync(path.dirname(p), { recursive: true });
279
+
280
+ fs.closeSync(fs.openSync(p, "wx"));
281
+ const old = new Date(Date.now() - 10_000);
282
+ fs.utimesSync(p, old, old);
283
+ expect(() => acquireOperationLock(rootA, "sync", { wait: false })).toThrowError(
284
+ OperationLockedError,
285
+ );
286
+ expect(fs.readFileSync(p, "utf8")).toBe("");
287
+
288
+ fs.writeFileSync(p, "{ torn");
289
+ expect(() => acquireOperationLock(rootA, "sync", { wait: false })).toThrowError(
290
+ OperationLockedError,
291
+ );
292
+ expect(fs.readFileSync(p, "utf8")).toBe("{ torn");
234
293
  });
235
294
 
236
295
  it("different HQ roots are independent — both may hold concurrently", () => {
@@ -29,9 +29,10 @@
29
29
  *
30
30
  * ## Atomicity, liveness, takeover
31
31
  *
32
- * - Acquisition uses `open(…, "wx")` (O_CREAT | O_EXCL) an atomic
33
- * create-if-absent. Exactly one racer can create the file; the loser sees
34
- * EEXIST and re-evaluates.
32
+ * - Acquisition writes the full owner payload to a same-directory temp file,
33
+ * fsyncs it, then publishes that complete file into the final lock name
34
+ * with create-if-absent semantics. Exactly one racer can publish; the loser
35
+ * sees EEXIST and re-evaluates.
35
36
  * - The lock records the holder's `{ pid, command, startedAt, hqRoot }`. On
36
37
  * EEXIST we test the recorded PID with `process.kill(pid, 0)`:
37
38
  * * ESRCH → the holder is gone (crashed / killed -9 / stale file) →
@@ -65,11 +66,10 @@
65
66
  * holder is reclaimed at once regardless of the wait config.
66
67
  *
67
68
  * Ordering / scope caveats:
68
- * - This is a CROSS-PROCESS mutex keyed on the holder's PID. Two concurrent
69
- * acquisitions inside the SAME process share a PID, so the same-process
70
- * reclaim path lets them stomp each other in-process concurrent acquire
71
- * is unsupported (the real consumers — sync / rescue / reindex — are
72
- * separate processes).
69
+ * - This is a CROSS-PROCESS mutex keyed on the holder's PID. In-process
70
+ * concurrent acquire is unsupported (the real consumers sync / rescue /
71
+ * reindex are separate processes), but a live same-PID holder is still
72
+ * treated as busy rather than reclaimed.
73
73
  * - When several distinct processes wait on the same lock, the next one to
74
74
  * win the O_EXCL race after a free acquires. Order is best-effort, NOT
75
75
  * FIFO — do not depend on arrival order.
@@ -235,9 +235,18 @@ function stateDir(): string {
235
235
  return process.env.HQ_STATE_DIR || path.join(os.homedir(), ".hq");
236
236
  }
237
237
 
238
+ function canonicalRoot(hqRoot: string): string {
239
+ const resolved = path.resolve(hqRoot);
240
+ try {
241
+ return fs.realpathSync.native(resolved);
242
+ } catch {
243
+ return resolved;
244
+ }
245
+ }
246
+
238
247
  /** Absolute lock path for a given HQ root. Exported for tests. */
239
248
  export function lockPathFor(hqRoot: string): string {
240
- const canon = path.resolve(hqRoot);
249
+ const canon = canonicalRoot(hqRoot);
241
250
  const key = crypto.createHash("sha1").update(canon).digest("hex").slice(0, 16);
242
251
  return path.join(stateDir(), "locks", `operation-${key}.lock`);
243
252
  }
@@ -271,6 +280,22 @@ function readLockInfo(p: string): LockInfo | null {
271
280
  }
272
281
  }
273
282
 
283
+ function initializingLockInfo(p: string): LockInfo {
284
+ let mtimeMs = Date.now();
285
+ try {
286
+ const st = fs.statSync(p);
287
+ mtimeMs = st.mtimeMs;
288
+ } catch {
289
+ /* lock disappeared between EEXIST and stat; treat as transient busy */
290
+ }
291
+ return {
292
+ pid: 0,
293
+ command: "operation-lock writer",
294
+ startedAt: new Date(mtimeMs).toISOString(),
295
+ hqRoot: "unknown",
296
+ };
297
+ }
298
+
274
299
  // ── Process-wide release plumbing ──────────────────────────────────────────
275
300
  // Track every lock this process currently holds so the signal/exit hooks can
276
301
  // release all of them. The hooks are installed exactly once.
@@ -333,7 +358,7 @@ function disabledHandle(hqRoot: string, command: string): LockHandle {
333
358
  pid: process.pid,
334
359
  command,
335
360
  startedAt: new Date().toISOString(),
336
- hqRoot: path.resolve(hqRoot),
361
+ hqRoot: canonicalRoot(hqRoot),
337
362
  };
338
363
  return { ...NOOP_HANDLE_BASE, path: "", info };
339
364
  }
@@ -346,17 +371,68 @@ function prepareLock(hqRoot: string, command: string): { p: string; info: LockIn
346
371
  pid: process.pid,
347
372
  command,
348
373
  startedAt: new Date().toISOString(),
349
- hqRoot: path.resolve(hqRoot),
374
+ hqRoot: canonicalRoot(hqRoot),
350
375
  };
351
376
  return { p, info, payload: JSON.stringify(info, null, 2) };
352
377
  }
353
378
 
379
+ function writeLockTemp(dir: string, payload: string): string {
380
+ const tmp = path.join(
381
+ dir,
382
+ `.operation-lock.${process.pid}.${Date.now()}.${crypto.randomBytes(6).toString("hex")}.tmp`,
383
+ );
384
+ const fd = fs.openSync(tmp, "wx", 0o600);
385
+ let closed = false;
386
+ try {
387
+ fs.writeSync(fd, payload);
388
+ fs.fsyncSync(fd);
389
+ fs.closeSync(fd);
390
+ closed = true;
391
+ return tmp;
392
+ } catch (err) {
393
+ if (!closed) {
394
+ try {
395
+ fs.closeSync(fd);
396
+ } catch {
397
+ /* best-effort cleanup */
398
+ }
399
+ }
400
+ try {
401
+ fs.rmSync(tmp, { force: true });
402
+ } catch {
403
+ /* best-effort cleanup */
404
+ }
405
+ throw err;
406
+ }
407
+ }
408
+
409
+ function publishLockPayload(p: string, payload: string): boolean {
410
+ const dir = path.dirname(p);
411
+ const tmp = writeLockTemp(dir, payload);
412
+ try {
413
+ // `link` is the no-overwrite atomic publish primitive available in Node:
414
+ // it fails with EEXIST if the final lock name is already present, and the
415
+ // final name never exists until the fully-written payload is linked there.
416
+ fs.linkSync(tmp, p);
417
+ return true;
418
+ } catch (err) {
419
+ if ((err as NodeJS.ErrnoException)?.code === "EEXIST") return false;
420
+ throw err;
421
+ } finally {
422
+ try {
423
+ fs.rmSync(tmp, { force: true });
424
+ } catch {
425
+ /* best-effort cleanup */
426
+ }
427
+ }
428
+ }
429
+
354
430
  /**
355
431
  * One acquisition pass. Returns the {@link LockHandle} on success, or
356
432
  * `{ busy }` naming the LIVE holder that blocked us (so the caller can decide
357
- * to wait or refuse). A stale/torn/own-leftover lock is reclaimed in-pass and
358
- * never reported as busy. Throws only on genuinely pathological churn or a
359
- * non-EEXIST fs error.
433
+ * to wait or refuse). A valid stale lock is reclaimed in-pass and never
434
+ * reported as busy; empty/torn locks are treated as initializing and left in
435
+ * place. Throws only on genuinely pathological churn or unexpected fs errors.
360
436
  */
361
437
  function tryAcquireOnce(
362
438
  p: string,
@@ -368,30 +444,22 @@ function tryAcquireOnce(
368
444
  // it as busy.
369
445
  const MAX_ATTEMPTS = 5;
370
446
  for (let attempt = 0; attempt < MAX_ATTEMPTS; attempt++) {
371
- let fd: number;
372
- try {
373
- fd = fs.openSync(p, "wx"); // O_CREAT | O_EXCL — atomic
374
- } catch (err) {
375
- if ((err as NodeJS.ErrnoException)?.code !== "EEXIST") throw err;
447
+ if (publishLockPayload(p, payload)) {
448
+ return { handle: makeHandle(p, info) };
449
+ }
376
450
 
377
- const holder = readLockInfo(p);
378
- if (holder && holder.pid !== process.pid && pidAlive(holder.pid)) {
379
- return { busy: holder };
380
- }
381
- // Stale (dead holder), unreadable/torn, or our own leftover → reclaim.
451
+ const holder = readLockInfo(p);
452
+ if (holder) {
453
+ if (pidAlive(holder.pid)) return { busy: holder };
382
454
  try {
383
455
  fs.unlinkSync(p);
384
456
  } catch {
385
- /* someone else reclaimed it first; the next openSync re-evaluates */
457
+ /* someone else reclaimed it first; the next publish re-evaluates */
386
458
  }
387
459
  continue;
388
460
  }
389
- try {
390
- fs.writeSync(fd, payload);
391
- } finally {
392
- fs.closeSync(fd);
393
- }
394
- return { handle: makeHandle(p, info) };
461
+
462
+ return { busy: initializingLockInfo(p) };
395
463
  }
396
464
 
397
465
  // Pathological churn (another process reclaiming in lockstep). Surface it
@@ -523,6 +523,48 @@ describe("personal-vault: continuity-pointer carve-out", () => {
523
523
  path.join(THREADS, "handoff.json"),
524
524
  ]);
525
525
  });
526
+
527
+ it("R-F35: rejects threads symlinks below hqRoot while allowing symlinked hqRoot ancestors", () => {
528
+ const privateThreads = path.join(hqRoot, "repos", "private");
529
+ fs.mkdirSync(privateThreads, { recursive: true });
530
+ fs.mkdirSync(path.join(hqRoot, "workspace"), { recursive: true });
531
+ fs.writeFileSync(path.join(privateThreads, "handoff.json"), JSON.stringify({
532
+ thread_path: "workspace/threads/T-private.json",
533
+ }));
534
+ fs.writeFileSync(path.join(privateThreads, "T-private.json"), "{}");
535
+ fs.symlinkSync(privateThreads, path.join(hqRoot, THREADS), "dir");
536
+
537
+ expect(rel(computeContinuityPointerPaths(hqRoot))).toEqual([
538
+ path.join(THREADS, "handoff.json"),
539
+ ]);
540
+
541
+ const realParent = fs.mkdtempSync(path.join(os.tmpdir(), "hq-cont-real-"));
542
+ const aliasParent = fs.mkdtempSync(path.join(os.tmpdir(), "hq-cont-alias-"));
543
+ const realRoot = path.join(realParent, "root");
544
+ const aliasRoot = path.join(aliasParent, "root-link");
545
+ try {
546
+ fs.mkdirSync(path.join(realRoot, THREADS), { recursive: true });
547
+ fs.writeFileSync(
548
+ path.join(realRoot, THREADS, "handoff.json"),
549
+ JSON.stringify({ thread_path: "workspace/threads/T-active.json" }),
550
+ );
551
+ fs.writeFileSync(path.join(realRoot, THREADS, "T-active.json"), "{}");
552
+ fs.symlinkSync(realRoot, aliasRoot, "dir");
553
+
554
+ const aliasRel = computeContinuityPointerPaths(aliasRoot)
555
+ .map((p) => path.relative(aliasRoot, p))
556
+ .sort();
557
+ expect(aliasRel).toEqual(
558
+ [
559
+ path.join(THREADS, "T-active.json"),
560
+ path.join(THREADS, "handoff.json"),
561
+ ].sort(),
562
+ );
563
+ } finally {
564
+ fs.rmSync(realParent, { recursive: true, force: true });
565
+ fs.rmSync(aliasParent, { recursive: true, force: true });
566
+ }
567
+ });
526
568
  });
527
569
 
528
570
  // ─────────────────────────────────────────────────────────────────────────