@remnic/core 1.1.29 → 1.1.31

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (100) hide show
  1. package/dist/access-cli.js +13 -13
  2. package/dist/access-http.d.ts +2 -1
  3. package/dist/access-http.js +8 -8
  4. package/dist/access-mcp.d.ts +1 -1
  5. package/dist/access-mcp.js +7 -7
  6. package/dist/access-schema.d.ts +55 -5
  7. package/dist/access-schema.js +4 -2
  8. package/dist/{access-service-CEyV8XJ5.d.ts → access-service-CkZyb35d.d.ts} +10 -2
  9. package/dist/access-service.d.ts +1 -1
  10. package/dist/access-service.js +5 -5
  11. package/dist/briefing.js +2 -2
  12. package/dist/causal-consolidation.js +3 -3
  13. package/dist/{chunk-25YQM6XW.js → chunk-2IWUMAES.js} +3 -3
  14. package/dist/{chunk-6CB4E7ZV.js → chunk-3ZLVGM76.js} +4 -4
  15. package/dist/{chunk-QYHQ2JHL.js → chunk-43PJZYGL.js} +2 -2
  16. package/dist/{chunk-YITUHONZ.js → chunk-4KGVTPGD.js} +2 -2
  17. package/dist/{chunk-TR4DK5OH.js → chunk-76FLAAUC.js} +2 -2
  18. package/dist/{chunk-6BFAEWQS.js → chunk-77H5NU3M.js} +2 -2
  19. package/dist/{chunk-IANK6Y5W.js → chunk-A6KTB5R6.js} +2 -2
  20. package/dist/{chunk-7D6O46PF.js → chunk-BVF3AGJP.js} +2 -2
  21. package/dist/{chunk-4H6DURG6.js → chunk-JA3AK3PT.js} +2 -2
  22. package/dist/{chunk-RCZRL5BE.js → chunk-MRILGULB.js} +2 -2
  23. package/dist/{chunk-CWWDIQZB.js → chunk-QLLBRHAT.js} +8 -8
  24. package/dist/{chunk-2WIPXV3Y.js → chunk-RR2PKP3I.js} +2 -2
  25. package/dist/{chunk-3F24QTRI.js → chunk-SAZS2QZB.js} +2 -2
  26. package/dist/{chunk-VYU7PXUS.js → chunk-SIC6U3GZ.js} +2 -2
  27. package/dist/{chunk-WDSIV3AK.js → chunk-TPU5L5EY.js} +12 -12
  28. package/dist/{chunk-AMVN77EU.js → chunk-U7EJOMFC.js} +371 -91
  29. package/dist/chunk-U7EJOMFC.js.map +1 -0
  30. package/dist/{chunk-F33CJ5CH.js → chunk-VBJ7V5SK.js} +40 -8
  31. package/dist/chunk-VBJ7V5SK.js.map +1 -0
  32. package/dist/{chunk-6WV2HYTZ.js → chunk-W6AQJ2PY.js} +4 -4
  33. package/dist/{chunk-PUXCIHRL.js → chunk-XSZEP4SF.js} +2 -2
  34. package/dist/{chunk-NW7JW5GA.js → chunk-YROHKYBY.js} +41 -6
  35. package/dist/chunk-YROHKYBY.js.map +1 -0
  36. package/dist/{chunk-JUYT2J3K.js → chunk-YU5KIWYQ.js} +136 -8
  37. package/dist/chunk-YU5KIWYQ.js.map +1 -0
  38. package/dist/{chunk-LCTP7YRU.js → chunk-ZAVUCJ4H.js} +38 -7
  39. package/dist/chunk-ZAVUCJ4H.js.map +1 -0
  40. package/dist/{cli-BguVmIwO.d.ts → cli-kuh9PwZ5.d.ts} +1 -1
  41. package/dist/cli.d.ts +2 -2
  42. package/dist/cli.js +17 -17
  43. package/dist/compounding/engine.js +2 -2
  44. package/dist/connectors/codex-materialize-runner.js +2 -2
  45. package/dist/connectors/index.js +2 -2
  46. package/dist/entity-retrieval.js +2 -2
  47. package/dist/index.d.ts +4 -4
  48. package/dist/index.js +34 -22
  49. package/dist/index.js.map +1 -1
  50. package/dist/maintenance/memory-governance.js +2 -2
  51. package/dist/maintenance/rebuild-memory-lifecycle-ledger.js +2 -2
  52. package/dist/maintenance/rebuild-memory-projection.js +3 -3
  53. package/dist/mcp-memory-inspector-app.d.ts +1 -1
  54. package/dist/namespaces/migrate.js +3 -3
  55. package/dist/namespaces/storage.js +2 -2
  56. package/dist/offline-sync.d.ts +56 -1
  57. package/dist/offline-sync.js +15 -1
  58. package/dist/operator-toolkit.js +5 -5
  59. package/dist/orchestrator.js +9 -9
  60. package/dist/schemas.d.ts +22 -22
  61. package/dist/semantic-consolidation.js +3 -3
  62. package/dist/semantic-rule-promotion.js +2 -2
  63. package/dist/semantic-rule-verifier.js +2 -2
  64. package/dist/storage.d.ts +5 -0
  65. package/dist/storage.js +1 -1
  66. package/dist/transfer/types.d.ts +12 -12
  67. package/dist/verified-recall.js +2 -2
  68. package/package.json +1 -1
  69. package/src/access-http.test.ts +355 -0
  70. package/src/access-http.ts +149 -1
  71. package/src/access-schema.ts +58 -3
  72. package/src/access-service-namespace.test.ts +56 -1
  73. package/src/access-service-offline-file-content.test.ts +17 -0
  74. package/src/access-service.ts +47 -1
  75. package/src/index.ts +7 -0
  76. package/src/offline-sync.test.ts +1055 -1
  77. package/src/offline-sync.ts +465 -97
  78. package/src/storage.ts +36 -2
  79. package/dist/chunk-AMVN77EU.js.map +0 -1
  80. package/dist/chunk-F33CJ5CH.js.map +0 -1
  81. package/dist/chunk-JUYT2J3K.js.map +0 -1
  82. package/dist/chunk-LCTP7YRU.js.map +0 -1
  83. package/dist/chunk-NW7JW5GA.js.map +0 -1
  84. /package/dist/{chunk-25YQM6XW.js.map → chunk-2IWUMAES.js.map} +0 -0
  85. /package/dist/{chunk-6CB4E7ZV.js.map → chunk-3ZLVGM76.js.map} +0 -0
  86. /package/dist/{chunk-QYHQ2JHL.js.map → chunk-43PJZYGL.js.map} +0 -0
  87. /package/dist/{chunk-YITUHONZ.js.map → chunk-4KGVTPGD.js.map} +0 -0
  88. /package/dist/{chunk-TR4DK5OH.js.map → chunk-76FLAAUC.js.map} +0 -0
  89. /package/dist/{chunk-6BFAEWQS.js.map → chunk-77H5NU3M.js.map} +0 -0
  90. /package/dist/{chunk-IANK6Y5W.js.map → chunk-A6KTB5R6.js.map} +0 -0
  91. /package/dist/{chunk-7D6O46PF.js.map → chunk-BVF3AGJP.js.map} +0 -0
  92. /package/dist/{chunk-4H6DURG6.js.map → chunk-JA3AK3PT.js.map} +0 -0
  93. /package/dist/{chunk-RCZRL5BE.js.map → chunk-MRILGULB.js.map} +0 -0
  94. /package/dist/{chunk-CWWDIQZB.js.map → chunk-QLLBRHAT.js.map} +0 -0
  95. /package/dist/{chunk-2WIPXV3Y.js.map → chunk-RR2PKP3I.js.map} +0 -0
  96. /package/dist/{chunk-3F24QTRI.js.map → chunk-SAZS2QZB.js.map} +0 -0
  97. /package/dist/{chunk-VYU7PXUS.js.map → chunk-SIC6U3GZ.js.map} +0 -0
  98. /package/dist/{chunk-WDSIV3AK.js.map → chunk-TPU5L5EY.js.map} +0 -0
  99. /package/dist/{chunk-6WV2HYTZ.js.map → chunk-W6AQJ2PY.js.map} +0 -0
  100. /package/dist/{chunk-PUXCIHRL.js.map → chunk-XSZEP4SF.js.map} +0 -0
@@ -5,6 +5,7 @@ import { existsSync } from "node:fs";
5
5
  import { readFile } from "node:fs/promises";
6
6
  import path from "node:path";
7
7
  import { fileURLToPath, URL } from "node:url";
8
+ import { gunzipSync } from "node:zlib";
8
9
  import { log } from "./logger.js";
9
10
  import { EngramAccessInputError, type EngramAccessService } from "./access-service.js";
10
11
  import { EngramMcpServer } from "./access-mcp.js";
@@ -12,6 +13,7 @@ import { validateRequest, type SchemaName, type SchemaTypeFor } from "./access-s
12
13
  import {
13
14
  OFFLINE_SYNC_APPLY_MAX_BODY_BYTES,
14
15
  OFFLINE_SYNC_FILE_CONTENT_MAX_CHUNK_BYTES,
16
+ OFFLINE_SYNC_SNAPSHOT_BASE_MAX_BODY_BYTES,
15
17
  } from "./offline-sync.js";
16
18
  import type { RecallDisclosure, RecallPlanMode } from "./types.js";
17
19
  import { isRecallDisclosure } from "./types.js";
@@ -125,6 +127,16 @@ function parseTrustZoneFilter(raw: string | null): TrustZoneName | undefined {
125
127
  throw new HttpError(400, "zone must be one of quarantine|working|trusted", "invalid_zone_filter");
126
128
  }
127
129
 
130
+ function summarizeHttpRequest(req: IncomingMessage): string {
131
+ const method = req.method ?? "UNKNOWN";
132
+ try {
133
+ const parsed = new URL(req.url ?? "/", "http://localhost");
134
+ return `${method} ${parsed.pathname}`;
135
+ } catch {
136
+ return `${method} ${(req.url ?? "/").split("?")[0]}`;
137
+ }
138
+ }
139
+
128
140
  /**
129
141
  * Decode a `:peerId` URL path segment, converting malformed percent-encoded
130
142
  * input (e.g., `%E0%A4%A`) into a 400 client error rather than letting
@@ -217,6 +229,10 @@ export class EngramAccessHttpServer {
217
229
  res.destroy(err as Error);
218
230
  return;
219
231
  }
232
+ log.error(
233
+ `engram access HTTP internal error [${correlationId}] ${summarizeHttpRequest(req)}`,
234
+ err,
235
+ );
220
236
  this.respondJson(res, 500, { error: "internal_error", code: "internal_error" });
221
237
  });
222
238
  });
@@ -630,6 +646,63 @@ export class EngramAccessHttpServer {
630
646
  return;
631
647
  }
632
648
 
649
+ if (
650
+ req.method === "GET" &&
651
+ (pathname === "/engram/v1/offline-sync/snapshot-stream" ||
652
+ pathname === "/remnic/v1/offline-sync/snapshot-stream")
653
+ ) {
654
+ const includeTranscriptsRaw = parsed.searchParams.get("include_transcripts");
655
+ const includeContentRaw = parsed.searchParams.get("content");
656
+ if (
657
+ includeTranscriptsRaw !== null &&
658
+ includeTranscriptsRaw !== "true" &&
659
+ includeTranscriptsRaw !== "false"
660
+ ) {
661
+ throw new EngramAccessInputError(
662
+ `include_transcripts must be one of: true, false (got: ${includeTranscriptsRaw})`,
663
+ );
664
+ }
665
+ if (
666
+ includeContentRaw !== null &&
667
+ includeContentRaw !== "false"
668
+ ) {
669
+ throw new EngramAccessInputError("snapshot-stream content must be false");
670
+ }
671
+ const namespaceParam = parsed.searchParams.get("namespace");
672
+ const result = await this.service.offlineSyncSnapshotStream({
673
+ namespace: this.resolveNamespace(
674
+ req,
675
+ namespaceParam && namespaceParam.length > 0 ? namespaceParam : undefined,
676
+ ),
677
+ principal: this.resolveRequestPrincipal(req),
678
+ includeTranscripts: includeTranscriptsRaw !== "false",
679
+ includeContent: false,
680
+ });
681
+ await this.respondOfflineSnapshotStream(res, result);
682
+ return;
683
+ }
684
+
685
+ if (
686
+ req.method === "POST" &&
687
+ (pathname === "/engram/v1/offline-sync/snapshot" || pathname === "/remnic/v1/offline-sync/snapshot")
688
+ ) {
689
+ const body = await this.readValidatedBody(
690
+ req,
691
+ "offlineSyncSnapshot",
692
+ OFFLINE_SYNC_SNAPSHOT_BASE_MAX_BODY_BYTES,
693
+ );
694
+ const result = await this.service.offlineSyncSnapshot({
695
+ namespace: this.resolveNamespace(req, body.namespace),
696
+ principal: this.resolveRequestPrincipal(req),
697
+ includeTranscripts: body.includeTranscripts,
698
+ includeContent: body.includeContent,
699
+ baseFiles: body.baseFiles,
700
+ ...(body.baseCapturedAt ? { baseCapturedAt: new Date(body.baseCapturedAt) } : {}),
701
+ });
702
+ this.respondJson(res, 200, result);
703
+ return;
704
+ }
705
+
633
706
  if (
634
707
  req.method === "POST" &&
635
708
  (pathname === "/engram/v1/offline-sync/files" || pathname === "/remnic/v1/offline-sync/files")
@@ -717,6 +790,7 @@ export class EngramAccessHttpServer {
717
790
  namespace: this.resolveNamespace(req, body.namespace),
718
791
  principal: this.resolveRequestPrincipal(req),
719
792
  changeset: body.changeset,
793
+ returnCurrentFiles: body.returnCurrentFiles,
720
794
  });
721
795
  this.respondJson(res, 200, result);
722
796
  return;
@@ -1868,6 +1942,62 @@ export class EngramAccessHttpServer {
1868
1942
  res.end(body);
1869
1943
  }
1870
1944
 
1945
+ private async respondOfflineSnapshotStream(
1946
+ res: ServerResponse,
1947
+ snapshot: Awaited<ReturnType<EngramAccessService["offlineSyncSnapshotStream"]>>,
1948
+ ): Promise<void> {
1949
+ res.statusCode = 200;
1950
+ res.setHeader("content-type", "application/x-ndjson; charset=utf-8");
1951
+ res.setHeader("cache-control", "no-store");
1952
+ const cid = correlationIdStore.getStore();
1953
+ if (cid) {
1954
+ res.setHeader("x-request-id", cid);
1955
+ }
1956
+ const waitForDrainOrClose = async (): Promise<boolean> => new Promise((resolve, reject) => {
1957
+ const cleanup = () => {
1958
+ res.off("drain", onDrain);
1959
+ res.off("close", onClose);
1960
+ res.off("error", onError);
1961
+ };
1962
+ const onDrain = () => {
1963
+ cleanup();
1964
+ resolve(true);
1965
+ };
1966
+ const onClose = () => {
1967
+ cleanup();
1968
+ resolve(false);
1969
+ };
1970
+ const onError = (error: Error) => {
1971
+ cleanup();
1972
+ reject(error);
1973
+ };
1974
+ res.once("drain", onDrain);
1975
+ res.once("close", onClose);
1976
+ res.once("error", onError);
1977
+ });
1978
+ const writeLine = async (payload: unknown): Promise<boolean> => {
1979
+ if (res.destroyed || res.writableEnded) return false;
1980
+ if (res.write(`${JSON.stringify(payload)}\n`)) return true;
1981
+ if (res.destroyed || res.writableEnded) return false;
1982
+ return waitForDrainOrClose();
1983
+ };
1984
+ if (!await writeLine({
1985
+ type: "snapshot",
1986
+ namespace: snapshot.namespace,
1987
+ format: snapshot.format,
1988
+ schemaVersion: snapshot.schemaVersion,
1989
+ createdAt: snapshot.createdAt,
1990
+ sourceId: snapshot.sourceId,
1991
+ includeTranscripts: snapshot.includeTranscripts,
1992
+ })) return;
1993
+ for await (const file of snapshot.files) {
1994
+ if (!await writeLine({ type: "file", file })) return;
1995
+ }
1996
+ if (!res.destroyed && !res.writableEnded) {
1997
+ res.end();
1998
+ }
1999
+ }
2000
+
1871
2001
  private respondBinary(
1872
2002
  res: ServerResponse,
1873
2003
  status: number,
@@ -1926,6 +2056,10 @@ export class EngramAccessHttpServer {
1926
2056
  req: IncomingMessage,
1927
2057
  maxBodyBytes = this.maxBodyBytes,
1928
2058
  ): Promise<Record<string, unknown>> {
2059
+ const encoding = (this.readOptionalHeader(req, "content-encoding") ?? "identity").toLowerCase();
2060
+ if (encoding !== "identity" && encoding !== "gzip") {
2061
+ throw new HttpError(415, "unsupported_content_encoding", "unsupported_content_encoding");
2062
+ }
1929
2063
  const chunks: Buffer[] = [];
1930
2064
  let total = 0;
1931
2065
  for await (const chunk of req) {
@@ -1937,7 +2071,21 @@ export class EngramAccessHttpServer {
1937
2071
  chunks.push(buffer);
1938
2072
  }
1939
2073
  if (chunks.length === 0) return {};
1940
- const raw = Buffer.concat(chunks).toString("utf-8").trim();
2074
+ let body = Buffer.concat(chunks, total);
2075
+ if (encoding === "gzip") {
2076
+ try {
2077
+ body = gunzipSync(body, { maxOutputLength: maxBodyBytes });
2078
+ } catch (error) {
2079
+ if ((error as NodeJS.ErrnoException).code === "ERR_BUFFER_TOO_LARGE") {
2080
+ throw new HttpError(413, "request_body_too_large", "request_body_too_large");
2081
+ }
2082
+ throw new HttpError(400, "invalid_gzip_body", "invalid_gzip_body");
2083
+ }
2084
+ if (body.byteLength > maxBodyBytes) {
2085
+ throw new HttpError(413, "request_body_too_large", "request_body_too_large");
2086
+ }
2087
+ }
2088
+ const raw = body.toString("utf-8").trim();
1941
2089
  if (raw.length === 0) return {};
1942
2090
  let parsed: unknown;
1943
2091
  try {
@@ -9,8 +9,12 @@ import {
9
9
  ACTION_CONFIDENCE_RULE_KINDS,
10
10
  } from "./action-confidence.js";
11
11
  import { isValidCapsuleSince } from "./transfer/capsule-export.js";
12
+ import { validateArchiveRelativePath } from "./transfer/fs-utils.js";
12
13
  import { CAPSULE_ID_PATTERN } from "./transfer/types.js";
13
- import { OFFLINE_SYNC_FILE_CONTENT_MAX_CHUNK_BYTES } from "./offline-sync.js";
14
+ import {
15
+ OFFLINE_SYNC_FILE_CONTENT_MAX_CHUNK_BYTES,
16
+ OFFLINE_SYNC_MAX_MTIME_MS,
17
+ } from "./offline-sync.js";
14
18
 
15
19
  // ---------------------------------------------------------------------------
16
20
  // Error formatting
@@ -369,10 +373,57 @@ export const capsuleListRequestSchema = z
369
373
  // Offline sync
370
374
  // ---------------------------------------------------------------------------
371
375
 
376
+ function isValidOfflineSyncPath(value: string): boolean {
377
+ try {
378
+ validateArchiveRelativePath(value, "path");
379
+ return true;
380
+ } catch {
381
+ return false;
382
+ }
383
+ }
384
+
385
+ const offlineSyncPathSchema = z
386
+ .string()
387
+ .trim()
388
+ .min(1, "path must be non-empty")
389
+ .max(4096)
390
+ .refine(
391
+ isValidOfflineSyncPath,
392
+ "path must be a POSIX relative path without unsafe segments",
393
+ );
394
+
395
+ const offlineSyncFileStateSchema = z.object({
396
+ path: offlineSyncPathSchema,
397
+ sha256: z.string().regex(/^[a-f0-9]{64}$/i, "sha256 must be a 64-character hex digest"),
398
+ bytes: z.number().int().min(0),
399
+ mtimeMs: z.number().finite().min(0).max(OFFLINE_SYNC_MAX_MTIME_MS),
400
+ });
401
+
402
+ const offlineSyncBaseCapturedAtSchema = z
403
+ .string()
404
+ .trim()
405
+ .min(1, "baseCapturedAt must be non-empty when provided")
406
+ .max(64)
407
+ .refine((value) => Number.isFinite(Date.parse(value)), {
408
+ message: "baseCapturedAt must be a valid ISO 8601 timestamp",
409
+ });
410
+
411
+ export const offlineSyncSnapshotRequestSchema = z.object({
412
+ namespace: namespaceSchema,
413
+ includeTranscripts: z.boolean().optional(),
414
+ includeContent: z.boolean().optional(),
415
+ baseCapturedAt: offlineSyncBaseCapturedAtSchema.optional(),
416
+ baseFiles: z
417
+ .array(offlineSyncFileStateSchema)
418
+ .max(300_000, "baseFiles must contain 300000 or fewer entries")
419
+ .optional(),
420
+ });
421
+
372
422
  export const offlineSyncApplyRequestSchema = z
373
423
  .object({
374
424
  namespace: namespaceSchema,
375
425
  changeset: z.unknown(),
426
+ returnCurrentFiles: z.boolean().optional(),
376
427
  })
377
428
  .refine((value) => value.changeset !== undefined && value.changeset !== null, {
378
429
  message: "changeset is required",
@@ -383,14 +434,14 @@ export const offlineSyncFilesRequestSchema = z.object({
383
434
  namespace: namespaceSchema,
384
435
  includeTranscripts: z.boolean().optional(),
385
436
  paths: z
386
- .array(z.string().trim().min(1, "path must be non-empty").max(4096))
437
+ .array(offlineSyncPathSchema)
387
438
  .max(5000, "paths must contain 5000 or fewer entries"),
388
439
  });
389
440
 
390
441
  export const offlineSyncFileContentRequestSchema = z.object({
391
442
  namespace: namespaceSchema,
392
443
  includeTranscripts: z.boolean().optional(),
393
- path: z.string().trim().min(1, "path must be non-empty").max(4096),
444
+ path: offlineSyncPathSchema,
394
445
  offset: z.number().int().min(0).optional(),
395
446
  length: z.number().int().min(1).max(OFFLINE_SYNC_FILE_CONTENT_MAX_CHUNK_BYTES).optional(),
396
447
  });
@@ -461,6 +512,7 @@ export type CapsuleExportRequest = z.infer<typeof capsuleExportRequestSchema>;
461
512
  export type CapsuleImportRequest = z.infer<typeof capsuleImportRequestSchema>;
462
513
  export type CapsuleListRequest = z.infer<typeof capsuleListRequestSchema>;
463
514
  export type OfflineSyncApplyRequest = z.infer<typeof offlineSyncApplyRequestSchema>;
515
+ export type OfflineSyncSnapshotRequest = z.infer<typeof offlineSyncSnapshotRequestSchema>;
464
516
  export type OfflineSyncFilesRequest = z.infer<typeof offlineSyncFilesRequestSchema>;
465
517
  export type OfflineSyncFileContentRequest = z.infer<typeof offlineSyncFileContentRequestSchema>;
466
518
  export type ActionConfidenceRequest = z.infer<typeof actionConfidenceRequestSchema>;
@@ -486,6 +538,7 @@ export type SchemaName =
486
538
  | "capsuleExport"
487
539
  | "capsuleImport"
488
540
  | "capsuleList"
541
+ | "offlineSyncSnapshot"
489
542
  | "offlineSyncFiles"
490
543
  | "offlineSyncFileContent"
491
544
  | "offlineSyncApply"
@@ -508,6 +561,7 @@ export type SchemaTypeFor<N extends SchemaName> =
508
561
  : N extends "capsuleExport" ? CapsuleExportRequest
509
562
  : N extends "capsuleImport" ? CapsuleImportRequest
510
563
  : N extends "capsuleList" ? CapsuleListRequest
564
+ : N extends "offlineSyncSnapshot" ? OfflineSyncSnapshotRequest
511
565
  : N extends "offlineSyncFiles" ? OfflineSyncFilesRequest
512
566
  : N extends "offlineSyncFileContent" ? OfflineSyncFileContentRequest
513
567
  : N extends "offlineSyncApply" ? OfflineSyncApplyRequest
@@ -531,6 +585,7 @@ const schemas: Record<SchemaName, z.ZodTypeAny> = {
531
585
  capsuleExport: capsuleExportRequestSchema,
532
586
  capsuleImport: capsuleImportRequestSchema,
533
587
  capsuleList: capsuleListRequestSchema,
588
+ offlineSyncSnapshot: offlineSyncSnapshotRequestSchema,
534
589
  offlineSyncFiles: offlineSyncFilesRequestSchema,
535
590
  offlineSyncFileContent: offlineSyncFileContentRequestSchema,
536
591
  offlineSyncApply: offlineSyncApplyRequestSchema,
@@ -1,5 +1,6 @@
1
1
  import assert from "node:assert/strict";
2
- import { mkdtemp, rm, symlink } from "node:fs/promises";
2
+ import { createHash } from "node:crypto";
3
+ import { mkdir, mkdtemp, readFile, rm, symlink, utimes, writeFile } from "node:fs/promises";
3
4
  import os from "node:os";
4
5
  import path from "node:path";
5
6
  import test from "node:test";
@@ -184,3 +185,57 @@ test("offlineSyncFiles reports symlink requested paths as input errors", async (
184
185
  await rm(root, { recursive: true, force: true });
185
186
  }
186
187
  });
188
+
189
+ test("offlineSyncSnapshot ignores client capture time for server fast-base scans", async () => {
190
+ const root = await mkdtemp(path.join(os.tmpdir(), "remnic-offline-snapshot-client-clock-"));
191
+ try {
192
+ const relPath = "facts/a.md";
193
+ const filePath = path.join(root, relPath);
194
+ const oldContent = Buffer.from("alpha");
195
+ const newContent = Buffer.from("bravo");
196
+ const mtimeMs = 1_700_000_000_000;
197
+ await mkdir(path.dirname(filePath), { recursive: true });
198
+ await writeFile(filePath, oldContent);
199
+ await utimes(filePath, mtimeMs / 1000, mtimeMs / 1000);
200
+ const baseFile = {
201
+ path: relPath,
202
+ sha256: createHash("sha256").update(oldContent).digest("hex"),
203
+ bytes: oldContent.byteLength,
204
+ mtimeMs,
205
+ };
206
+ await writeFile(filePath, newContent);
207
+ await utimes(filePath, mtimeMs / 1000, mtimeMs / 1000);
208
+
209
+ const { service } = makeService();
210
+ (service as unknown as {
211
+ orchestrator: {
212
+ config: PluginConfig;
213
+ getStorage(namespace: string): Promise<StorageManager>;
214
+ };
215
+ }).orchestrator.getStorage = async () => ({
216
+ dir: root,
217
+ async readOfflineSyncFile(targetPath: string) {
218
+ return readFile(targetPath);
219
+ },
220
+ async digestOfflineSyncFile(targetPath: string) {
221
+ const content = await readFile(targetPath);
222
+ return {
223
+ sha256: createHash("sha256").update(content).digest("hex"),
224
+ bytes: content.byteLength,
225
+ };
226
+ },
227
+ } as unknown as StorageManager);
228
+
229
+ const snapshot = await service.offlineSyncSnapshot({
230
+ namespace: "team",
231
+ principal: "reader",
232
+ includeContent: false,
233
+ baseFiles: [baseFile],
234
+ baseCapturedAt: new Date(Date.now() + 60_000),
235
+ });
236
+
237
+ assert.equal(snapshot.files[0]?.sha256, createHash("sha256").update(newContent).digest("hex"));
238
+ } finally {
239
+ await rm(root, { recursive: true, force: true });
240
+ }
241
+ });
@@ -2,6 +2,7 @@ import assert from "node:assert/strict";
2
2
  import test from "node:test";
3
3
 
4
4
  import { EngramAccessInputError, EngramAccessService } from "./access-service.js";
5
+ import { OFFLINE_SYNC_MAX_MTIME_MS } from "./offline-sync.js";
5
6
 
6
7
  function createOfflineService(): EngramAccessService {
7
8
  return new EngramAccessService({
@@ -34,4 +35,20 @@ test("offline apply-file-content reports invalid metadata as input errors", asyn
34
35
  error instanceof EngramAccessInputError &&
35
36
  /sha256 must be a 64-character sha256/.test(error.message),
36
37
  );
38
+
39
+ await assert.rejects(
40
+ () => service.offlineSyncApplyFileContent({
41
+ includeTranscripts: true,
42
+ sourceId: "laptop",
43
+ path: "state/lcm.sqlite",
44
+ sha256: "a".repeat(64),
45
+ bytes: 0,
46
+ mtimeMs: OFFLINE_SYNC_MAX_MTIME_MS + 1,
47
+ offset: 0,
48
+ content: Buffer.alloc(0),
49
+ }),
50
+ (error) =>
51
+ error instanceof EngramAccessInputError &&
52
+ /mtimeMs must be within JavaScript Date range/.test(error.message),
53
+ );
37
54
  });
@@ -139,11 +139,16 @@ import {
139
139
  applyOfflineSyncFileContentChunk,
140
140
  applyOfflineSyncChangeset,
141
141
  buildOfflineSyncSnapshot,
142
+ buildOfflineSyncSnapshotFromBase,
142
143
  buildOfflineSyncSnapshotForPaths,
144
+ iterateOfflineSyncSnapshotFileRecords,
145
+ OFFLINE_SYNC_SNAPSHOT_FORMAT,
143
146
  readOfflineSyncFileContentChunk,
144
147
  type OfflineSyncApplyFileContentChunkResult,
145
148
  type OfflineSyncApplyChangesetResult,
149
+ type OfflineSyncFileRecord,
146
150
  type OfflineSyncFileContentChunk,
151
+ type OfflineSyncFileState,
147
152
  type OfflineSyncSnapshot,
148
153
  } from "./offline-sync.js";
149
154
  import {
@@ -619,6 +624,8 @@ export interface EngramAccessOfflineSyncSnapshotRequest {
619
624
  principal?: string;
620
625
  includeTranscripts?: boolean;
621
626
  includeContent?: boolean;
627
+ baseCapturedAt?: Date;
628
+ baseFiles?: OfflineSyncFileState[];
622
629
  }
623
630
 
624
631
  export interface EngramAccessOfflineSyncFilesRequest {
@@ -655,12 +662,18 @@ export interface EngramAccessOfflineSyncApplyRequest {
655
662
  namespace?: string;
656
663
  principal?: string;
657
664
  changeset: unknown;
665
+ returnCurrentFiles?: boolean;
658
666
  }
659
667
 
660
668
  export interface EngramAccessOfflineSyncSnapshotResponse extends OfflineSyncSnapshot {
661
669
  namespace: string;
662
670
  }
663
671
 
672
+ export interface EngramAccessOfflineSyncSnapshotStreamResponse extends Omit<OfflineSyncSnapshot, "files"> {
673
+ namespace: string;
674
+ files: AsyncIterable<OfflineSyncFileRecord>;
675
+ }
676
+
664
677
  export interface EngramAccessOfflineSyncFilesResponse extends OfflineSyncSnapshot {
665
678
  namespace: string;
666
679
  }
@@ -5602,12 +5615,19 @@ export class EngramAccessService {
5602
5615
  const resolvedNamespace = this.resolveReadableNamespace(options.namespace, options.principal);
5603
5616
  const storage = await this.orchestrator.getStorage(resolvedNamespace);
5604
5617
  const storageHash = createHash("sha256").update(storage.dir).digest("hex").slice(0, 16);
5605
- const snapshot = await buildOfflineSyncSnapshot({
5618
+ const snapshotBuilder = options.includeContent === false && options.baseFiles && options.baseFiles.length > 0
5619
+ ? buildOfflineSyncSnapshotFromBase
5620
+ : buildOfflineSyncSnapshot;
5621
+ const snapshot = await snapshotBuilder({
5606
5622
  root: storage.dir,
5607
5623
  sourceId: `remnic:${resolvedNamespace}:${storageHash}`,
5624
+ ...(options.baseFiles && options.baseFiles.length > 0 ? { baseFiles: options.baseFiles } : {}),
5625
+ // Client clocks are not authoritative for server-side ctime reuse. A
5626
+ // future client timestamp can hide same-size, preserved-mtime rewrites.
5608
5627
  includeContent: options.includeContent !== false,
5609
5628
  includeTranscripts: options.includeTranscripts !== false,
5610
5629
  readFile: async ({ filePath }) => storage.readOfflineSyncFile(filePath),
5630
+ readFileDigest: async ({ filePath }) => storage.digestOfflineSyncFile(filePath),
5611
5631
  });
5612
5632
  return {
5613
5633
  namespace: resolvedNamespace,
@@ -5615,6 +5635,29 @@ export class EngramAccessService {
5615
5635
  };
5616
5636
  }
5617
5637
 
5638
+ async offlineSyncSnapshotStream(
5639
+ options: Omit<EngramAccessOfflineSyncSnapshotRequest, "baseCapturedAt" | "baseFiles"> = {},
5640
+ ): Promise<EngramAccessOfflineSyncSnapshotStreamResponse> {
5641
+ const resolvedNamespace = this.resolveReadableNamespace(options.namespace, options.principal);
5642
+ const storage = await this.orchestrator.getStorage(resolvedNamespace);
5643
+ const storageHash = createHash("sha256").update(storage.dir).digest("hex").slice(0, 16);
5644
+ return {
5645
+ namespace: resolvedNamespace,
5646
+ format: OFFLINE_SYNC_SNAPSHOT_FORMAT,
5647
+ schemaVersion: 1,
5648
+ createdAt: new Date().toISOString(),
5649
+ sourceId: `remnic:${resolvedNamespace}:${storageHash}`,
5650
+ includeTranscripts: options.includeTranscripts !== false,
5651
+ files: iterateOfflineSyncSnapshotFileRecords({
5652
+ root: storage.dir,
5653
+ includeContent: options.includeContent === true,
5654
+ includeTranscripts: options.includeTranscripts !== false,
5655
+ readFile: async ({ filePath }) => storage.readOfflineSyncFile(filePath),
5656
+ readFileDigest: async ({ filePath }) => storage.digestOfflineSyncFile(filePath),
5657
+ }),
5658
+ };
5659
+ }
5660
+
5618
5661
  async offlineSyncFiles(
5619
5662
  options: EngramAccessOfflineSyncFilesRequest,
5620
5663
  ): Promise<EngramAccessOfflineSyncFilesResponse> {
@@ -5702,6 +5745,7 @@ export class EngramAccessService {
5702
5745
  content: options.content,
5703
5746
  includeTranscripts: options.includeTranscripts !== false,
5704
5747
  readFile: async ({ filePath }) => storage.readOfflineSyncFile(filePath),
5748
+ readFileDigest: async ({ filePath }) => storage.digestOfflineSyncFile(filePath),
5705
5749
  writeFile: async ({ filePath, content }) => storage.writeOfflineSyncFile(filePath, content),
5706
5750
  writeStagingFile: async ({ filePath, content }) => storage.writeOfflineSyncStagingFile(filePath, content),
5707
5751
  writeFileChunks: async ({ filePath, chunks }) => storage.writeOfflineSyncFileChunks(filePath, chunks),
@@ -5743,7 +5787,9 @@ export class EngramAccessService {
5743
5787
  const result = await applyOfflineSyncChangeset({
5744
5788
  root: storage.dir,
5745
5789
  changeset: options.changeset,
5790
+ returnCurrentFiles: options.returnCurrentFiles,
5746
5791
  readFile: async ({ filePath }) => storage.readOfflineSyncFile(filePath),
5792
+ readFileDigest: async ({ filePath }) => storage.digestOfflineSyncFile(filePath),
5747
5793
  writeFile: async ({ filePath, content }) => storage.writeOfflineSyncFile(filePath, content),
5748
5794
  deleteFile: async ({ filePath }) => storage.deleteOfflineSyncFile(filePath),
5749
5795
  });
package/src/index.ts CHANGED
@@ -683,23 +683,29 @@ export {
683
683
  OFFLINE_SYNC_CHANGESET_FORMAT,
684
684
  OFFLINE_SYNC_FILE_CONTENT_MAX_CHUNK_BYTES,
685
685
  OFFLINE_SYNC_FILE_CONTENT_TRANSFER_CHUNK_BYTES,
686
+ OFFLINE_SYNC_SNAPSHOT_BASE_MAX_BODY_BYTES,
686
687
  OFFLINE_SYNC_SNAPSHOT_FORMAT,
687
688
  OFFLINE_SYNC_STATE_VERSION,
688
689
  applyOfflineSyncFileContentChunk,
689
690
  applyOfflineSyncChangeset,
690
691
  applyOfflineSyncSnapshot,
691
692
  buildOfflineSyncChangeset,
693
+ buildOfflineSyncChangesetFromSnapshot,
692
694
  buildOfflineSyncSnapshot,
695
+ buildOfflineSyncSnapshotFromBase,
693
696
  buildOfflineSyncSnapshotForPaths,
694
697
  defaultOfflineSyncStatePath,
695
698
  fileStatesFromSnapshot,
699
+ iterateOfflineSyncSnapshotFileRecords,
696
700
  normalizeOfflineSyncChangeset,
697
701
  normalizeOfflineSyncSnapshot,
698
702
  offlineSyncStateFromSnapshot,
699
703
  readOfflineSyncFileContentChunk,
700
704
  readOfflineSyncState,
705
+ shouldPreferIncomingOfflineRuntimeFile,
701
706
  summarizeOfflineSyncChangeset,
702
707
  summarizeOfflineSyncPendingChanges,
708
+ summarizeOfflineSyncPendingFiles,
703
709
  writeOfflineSyncState,
704
710
  type OfflineSyncApplyFileContentChunkResult,
705
711
  type OfflineSyncApplyChangesetResult,
@@ -707,6 +713,7 @@ export {
707
713
  type OfflineSyncChange,
708
714
  type OfflineSyncChangeset,
709
715
  type OfflineSyncConflict,
716
+ type OfflineSyncFileDigest,
710
717
  type OfflineSyncFileRecord,
711
718
  type OfflineSyncFileContentChunk,
712
719
  type OfflineSyncFileState,