@indigoai-us/hq-cloud 6.16.35 → 6.16.37
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin/sync-runner-watch-loop.test.js +1 -2
- package/dist/bin/sync-runner-watch-loop.test.js.map +1 -1
- package/dist/bin/sync-runner.test.js +6 -24
- package/dist/bin/sync-runner.test.js.map +1 -1
- package/dist/cli/sync.d.ts.map +1 -1
- package/dist/cli/sync.js +8 -19
- package/dist/cli/sync.js.map +1 -1
- package/dist/cli/sync.test.js +265 -1
- package/dist/cli/sync.test.js.map +1 -1
- package/dist/company-vault-transport.d.ts +0 -2
- package/dist/company-vault-transport.d.ts.map +1 -1
- package/dist/company-vault-transport.js +0 -2
- package/dist/company-vault-transport.js.map +1 -1
- package/dist/index.d.ts +2 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -1
- package/dist/index.js.map +1 -1
- package/dist/journal-row-store.d.ts +54 -2
- package/dist/journal-row-store.d.ts.map +1 -1
- package/dist/journal-row-store.js +305 -11
- package/dist/journal-row-store.js.map +1 -1
- package/dist/journal-snapshot-decoder.d.ts +15 -0
- package/dist/journal-snapshot-decoder.d.ts.map +1 -0
- package/dist/journal-snapshot-decoder.js +371 -0
- package/dist/journal-snapshot-decoder.js.map +1 -0
- package/dist/journal-snapshot-memory-helpers.d.ts +30 -0
- package/dist/journal-snapshot-memory-helpers.d.ts.map +1 -0
- package/dist/journal-snapshot-memory-helpers.js +212 -0
- package/dist/journal-snapshot-memory-helpers.js.map +1 -0
- package/dist/journal-snapshot-memory.test.d.ts +2 -0
- package/dist/journal-snapshot-memory.test.d.ts.map +1 -0
- package/dist/journal-snapshot-memory.test.js +32 -0
- package/dist/journal-snapshot-memory.test.js.map +1 -0
- package/dist/journal-snapshot-rows.d.ts +7 -0
- package/dist/journal-snapshot-rows.d.ts.map +1 -0
- package/dist/journal-snapshot-rows.js +10 -0
- package/dist/journal-snapshot-rows.js.map +1 -0
- package/dist/journal-snapshot-v4-memory.test.d.ts +2 -0
- package/dist/journal-snapshot-v4-memory.test.d.ts.map +1 -0
- package/dist/journal-snapshot-v4-memory.test.js +24 -0
- package/dist/journal-snapshot-v4-memory.test.js.map +1 -0
- package/dist/journal-snapshot-v4-writer-memory.test.d.ts +2 -0
- package/dist/journal-snapshot-v4-writer-memory.test.d.ts.map +1 -0
- package/dist/journal-snapshot-v4-writer-memory.test.js +76 -0
- package/dist/journal-snapshot-v4-writer-memory.test.js.map +1 -0
- package/dist/journal-snapshot-v4-writer.test.d.ts +2 -0
- package/dist/journal-snapshot-v4-writer.test.d.ts.map +1 -0
- package/dist/journal-snapshot-v4-writer.test.js +253 -0
- package/dist/journal-snapshot-v4-writer.test.js.map +1 -0
- package/dist/journal-snapshot-v4.d.ts +54 -0
- package/dist/journal-snapshot-v4.d.ts.map +1 -0
- package/dist/journal-snapshot-v4.js +386 -0
- package/dist/journal-snapshot-v4.js.map +1 -0
- package/dist/journal-snapshot-v4.test.d.ts +2 -0
- package/dist/journal-snapshot-v4.test.d.ts.map +1 -0
- package/dist/journal-snapshot-v4.test.js +280 -0
- package/dist/journal-snapshot-v4.test.js.map +1 -0
- package/dist/journal-store.test.js +163 -2
- package/dist/journal-store.test.js.map +1 -1
- package/dist/journal.d.ts +26 -2
- package/dist/journal.d.ts.map +1 -1
- package/dist/journal.js +64 -6
- package/dist/journal.js.map +1 -1
- package/dist/journal.test.js +57 -49
- package/dist/journal.test.js.map +1 -1
- package/dist/manifest/build-manifest.bench.test.js +165 -58
- package/dist/manifest/build-manifest.bench.test.js.map +1 -1
- package/dist/skill-telemetry.d.ts +1 -2
- package/dist/skill-telemetry.d.ts.map +1 -1
- package/dist/skill-telemetry.js +1 -10
- package/dist/skill-telemetry.js.map +1 -1
- package/dist/sync/child-process-sync-worker-env.test.d.ts +2 -0
- package/dist/sync/child-process-sync-worker-env.test.d.ts.map +1 -0
- package/dist/sync/child-process-sync-worker-env.test.js +38 -0
- package/dist/sync/child-process-sync-worker-env.test.js.map +1 -0
- package/dist/sync/feature-flags.d.ts +0 -16
- package/dist/sync/feature-flags.d.ts.map +1 -1
- package/dist/sync/feature-flags.js +0 -11
- package/dist/sync/feature-flags.js.map +1 -1
- package/dist/sync/index.d.ts +2 -2
- package/dist/sync/index.d.ts.map +1 -1
- package/dist/sync/index.js +1 -1
- package/dist/sync/index.js.map +1 -1
- package/dist/sync/logger.test.js +1 -3
- package/dist/sync/logger.test.js.map +1 -1
- package/dist/sync/metrics.test.js +0 -3
- package/dist/sync/metrics.test.js.map +1 -1
- package/dist/sync/push-receiver.d.ts +0 -7
- package/dist/sync/push-receiver.d.ts.map +1 -1
- package/dist/sync/push-receiver.js.map +1 -1
- package/dist/sync/state-store.d.ts +43 -1
- package/dist/sync/state-store.d.ts.map +1 -1
- package/dist/sync/state-store.js +132 -55
- package/dist/sync/state-store.js.map +1 -1
- package/dist/sync/state-store.test.js +82 -0
- package/dist/sync/state-store.test.js.map +1 -1
- package/dist/watcher.test.js +7 -8
- package/dist/watcher.test.js.map +1 -1
- package/package.json +1 -1
- package/dist/daemon-worker.d.ts +0 -11
- package/dist/daemon-worker.d.ts.map +0 -1
- package/dist/daemon-worker.js +0 -22
- package/dist/daemon-worker.js.map +0 -1
package/dist/cli/sync.test.js
CHANGED
|
@@ -134,7 +134,7 @@ import { reindex } from "./reindex.js";
|
|
|
134
134
|
import { readSyncProgress } from "../sync-progress.js";
|
|
135
135
|
import { VaultAuthError } from "../vault-client.js";
|
|
136
136
|
import * as journalModule from "../journal.js";
|
|
137
|
-
import { readJournal, writeJournal, getEntry, listJournals } from "../journal.js";
|
|
137
|
+
import { readJournal, writeJournal, getEntry, listJournals, } from "../journal.js";
|
|
138
138
|
import { SqsPushReceiver } from "../sync/push-receiver.js";
|
|
139
139
|
import { buildTargetedPullArgv } from "../bin/sync-runner-watch-routes.js";
|
|
140
140
|
import { ReconcileCursorStore } from "../sync/reconcile-cursor.js";
|
|
@@ -245,6 +245,16 @@ function setupFetchMock(opts = {}) {
|
|
|
245
245
|
vi.stubGlobal("fetch", fetchMock);
|
|
246
246
|
return fetchMock;
|
|
247
247
|
}
|
|
248
|
+
function captureTargetedJournalDeltas() {
|
|
249
|
+
const deltas = [];
|
|
250
|
+
const originalApplyDelta = journalModule.JournalStore.prototype.applyDelta;
|
|
251
|
+
const applyDelta = vi.spyOn(journalModule.JournalStore.prototype, "applyDelta");
|
|
252
|
+
applyDelta.mockImplementation(function (delta) {
|
|
253
|
+
deltas.push(structuredClone(delta));
|
|
254
|
+
return originalApplyDelta.call(this, delta);
|
|
255
|
+
});
|
|
256
|
+
return { deltas, restore: () => applyDelta.mockRestore() };
|
|
257
|
+
}
|
|
248
258
|
describe("sync", () => {
|
|
249
259
|
let tmpDir;
|
|
250
260
|
let stateDir;
|
|
@@ -420,6 +430,260 @@ describe("sync", () => {
|
|
|
420
430
|
// expected "readJournal" to be called +0 times, but got 2 times
|
|
421
431
|
expect(materialisations).not.toHaveBeenCalled();
|
|
422
432
|
});
|
|
433
|
+
it("persists the current targeted-pull delta for changed, added, removed, and metadata rows", async () => {
|
|
434
|
+
const fixedNow = new Date("2026-09-11T12:00:00.000Z");
|
|
435
|
+
const oldChangedBody = "old changed body";
|
|
436
|
+
const removedBody = "removed body";
|
|
437
|
+
const oldChangedHash = crypto.createHash("sha256").update(oldChangedBody).digest("hex");
|
|
438
|
+
const removedHash = crypto.createHash("sha256").update(removedBody).digest("hex");
|
|
439
|
+
const companyRoot = path.join(tmpDir, "companies", "acme", "docs");
|
|
440
|
+
fs.mkdirSync(companyRoot, { recursive: true });
|
|
441
|
+
fs.writeFileSync(path.join(companyRoot, "changed.md"), oldChangedBody);
|
|
442
|
+
fs.writeFileSync(path.join(companyRoot, "removed.md"), removedBody);
|
|
443
|
+
writeJournal("acme", {
|
|
444
|
+
version: "2",
|
|
445
|
+
lastSync: "before-targeted-pull",
|
|
446
|
+
pulls: [],
|
|
447
|
+
files: {
|
|
448
|
+
"docs/changed.md": {
|
|
449
|
+
hash: oldChangedHash,
|
|
450
|
+
size: Buffer.byteLength(oldChangedBody),
|
|
451
|
+
syncedAt: "2026-09-10T00:00:00.000Z",
|
|
452
|
+
direction: "down",
|
|
453
|
+
remoteEtag: "old-etag",
|
|
454
|
+
},
|
|
455
|
+
"docs/removed.md": {
|
|
456
|
+
hash: removedHash,
|
|
457
|
+
size: Buffer.byteLength(removedBody),
|
|
458
|
+
syncedAt: "2026-09-10T00:00:00.000Z",
|
|
459
|
+
direction: "down",
|
|
460
|
+
remoteEtag: "removed-etag",
|
|
461
|
+
},
|
|
462
|
+
},
|
|
463
|
+
});
|
|
464
|
+
vi.mocked(remotePullModule.listRemoteForScope).mockResolvedValue([
|
|
465
|
+
{ key: "docs/changed.md", size: 17, lastModified: fixedNow, etag: '"new-etag"' },
|
|
466
|
+
{ key: "docs/added.md", size: 17, lastModified: fixedNow, etag: '"added-etag"' },
|
|
467
|
+
]);
|
|
468
|
+
const captured = captureTargetedJournalDeltas();
|
|
469
|
+
vi.useFakeTimers();
|
|
470
|
+
vi.setSystemTime(fixedNow);
|
|
471
|
+
try {
|
|
472
|
+
await sync({
|
|
473
|
+
company: "acme",
|
|
474
|
+
vaultConfig: mockConfig,
|
|
475
|
+
hqRoot: tmpDir,
|
|
476
|
+
pullPrefixes: ["docs"],
|
|
477
|
+
skipReindex: true,
|
|
478
|
+
onEvent: () => undefined,
|
|
479
|
+
});
|
|
480
|
+
}
|
|
481
|
+
finally {
|
|
482
|
+
vi.useRealTimers();
|
|
483
|
+
captured.restore();
|
|
484
|
+
}
|
|
485
|
+
expect(captured.deltas).toEqual([
|
|
486
|
+
{
|
|
487
|
+
upserts: {
|
|
488
|
+
"docs/changed.md": expect.objectContaining({
|
|
489
|
+
hash: "5633d479dfae75ba7a78914ee380fa202bd6126e7c6b7c22e3ebc9e1a6ddc871",
|
|
490
|
+
size: 17,
|
|
491
|
+
syncedAt: "2026-09-11T12:00:00.000Z",
|
|
492
|
+
direction: "down",
|
|
493
|
+
remoteEtag: "new-etag",
|
|
494
|
+
kind: "file",
|
|
495
|
+
mtimeMs: expect.any(Number),
|
|
496
|
+
ctimeMs: expect.any(Number),
|
|
497
|
+
}),
|
|
498
|
+
"docs/added.md": expect.objectContaining({
|
|
499
|
+
hash: "5633d479dfae75ba7a78914ee380fa202bd6126e7c6b7c22e3ebc9e1a6ddc871",
|
|
500
|
+
size: 17,
|
|
501
|
+
syncedAt: "2026-09-11T12:00:00.000Z",
|
|
502
|
+
direction: "down",
|
|
503
|
+
remoteEtag: "added-etag",
|
|
504
|
+
kind: "file",
|
|
505
|
+
mtimeMs: expect.any(Number),
|
|
506
|
+
ctimeMs: expect.any(Number),
|
|
507
|
+
}),
|
|
508
|
+
},
|
|
509
|
+
deletes: [],
|
|
510
|
+
metadata: {
|
|
511
|
+
lastSync: "2026-09-11T12:00:00.000Z",
|
|
512
|
+
pulls: [],
|
|
513
|
+
},
|
|
514
|
+
},
|
|
515
|
+
{
|
|
516
|
+
upserts: {},
|
|
517
|
+
deletes: ["docs/removed.md"],
|
|
518
|
+
metadata: {
|
|
519
|
+
lastSync: "2026-09-11T12:00:00.000Z",
|
|
520
|
+
pulls: [expect.objectContaining({
|
|
521
|
+
companyUid: "cmp_01ABCDEF",
|
|
522
|
+
startedAt: "2026-09-11T12:00:00.000Z",
|
|
523
|
+
completedAt: "2026-09-11T12:00:00.000Z",
|
|
524
|
+
syncMode: "all",
|
|
525
|
+
prefixSet: [""],
|
|
526
|
+
scopeChangeDetected: false,
|
|
527
|
+
orphansRemoved: 0,
|
|
528
|
+
orphansBlocked: 0,
|
|
529
|
+
})],
|
|
530
|
+
},
|
|
531
|
+
},
|
|
532
|
+
]);
|
|
533
|
+
});
|
|
534
|
+
it("re-anchors targeted-pull deltas at each download checkpoint", async () => {
|
|
535
|
+
const fixedNow = new Date("2026-09-11T12:00:00.000Z");
|
|
536
|
+
const remoteFiles = Array.from({ length: 10_001 }, (_value, index) => ({
|
|
537
|
+
key: `docs/checkpoint/${index}.md`,
|
|
538
|
+
size: 17,
|
|
539
|
+
lastModified: fixedNow,
|
|
540
|
+
etag: `"checkpoint-${index}"`,
|
|
541
|
+
}));
|
|
542
|
+
writeJournal("acme", { version: "2", lastSync: "", pulls: [], files: {} });
|
|
543
|
+
vi.mocked(remotePullModule.listRemoteForScope).mockResolvedValue(remoteFiles);
|
|
544
|
+
// The checkpoint contract is journal-only. Keep the 10,001 transfers at
|
|
545
|
+
// the real production batch boundary without adding 10,001 filesystem
|
|
546
|
+
// writes to this already broad sync suite.
|
|
547
|
+
const materializedPaths = new Set();
|
|
548
|
+
const originalLstatSync = fs.lstatSync;
|
|
549
|
+
const lstatSync = vi.spyOn(fs, "lstatSync").mockImplementation((filePath) => {
|
|
550
|
+
if (typeof filePath === "string" && materializedPaths.has(filePath)) {
|
|
551
|
+
return {
|
|
552
|
+
isSymbolicLink: () => false,
|
|
553
|
+
mtimeMs: fixedNow.getTime(),
|
|
554
|
+
ctimeMs: fixedNow.getTime(),
|
|
555
|
+
};
|
|
556
|
+
}
|
|
557
|
+
return originalLstatSync(filePath);
|
|
558
|
+
});
|
|
559
|
+
const downloadFile = vi.mocked(s3Module.downloadFile);
|
|
560
|
+
const originalDownload = downloadFile.getMockImplementation();
|
|
561
|
+
downloadFile.mockImplementation(async (_ctx, _key, localPath, options) => {
|
|
562
|
+
options?.beforeReplace?.();
|
|
563
|
+
materializedPaths.add(localPath);
|
|
564
|
+
return {
|
|
565
|
+
metadata: {},
|
|
566
|
+
contentHash: "5633d479dfae75ba7a78914ee380fa202bd6126e7c6b7c22e3ebc9e1a6ddc871",
|
|
567
|
+
contentSize: 17,
|
|
568
|
+
};
|
|
569
|
+
});
|
|
570
|
+
const captured = captureTargetedJournalDeltas();
|
|
571
|
+
vi.useFakeTimers();
|
|
572
|
+
vi.setSystemTime(fixedNow);
|
|
573
|
+
try {
|
|
574
|
+
await sync({
|
|
575
|
+
company: "acme",
|
|
576
|
+
vaultConfig: mockConfig,
|
|
577
|
+
hqRoot: tmpDir,
|
|
578
|
+
pullPrefixes: ["docs"],
|
|
579
|
+
skipReindex: true,
|
|
580
|
+
onEvent: () => undefined,
|
|
581
|
+
});
|
|
582
|
+
}
|
|
583
|
+
finally {
|
|
584
|
+
vi.useRealTimers();
|
|
585
|
+
captured.restore();
|
|
586
|
+
lstatSync.mockRestore();
|
|
587
|
+
if (originalDownload)
|
|
588
|
+
downloadFile.mockImplementation(originalDownload);
|
|
589
|
+
else
|
|
590
|
+
downloadFile.mockReset();
|
|
591
|
+
}
|
|
592
|
+
expect(captured.deltas).toHaveLength(3);
|
|
593
|
+
const [firstCheckpoint, secondCheckpoint, finalMetadata] = captured.deltas;
|
|
594
|
+
const firstKeys = Object.keys(firstCheckpoint.upserts ?? {});
|
|
595
|
+
const secondKeys = Object.keys(secondCheckpoint.upserts ?? {});
|
|
596
|
+
expect(firstKeys).toHaveLength(10_000);
|
|
597
|
+
expect(secondKeys).toHaveLength(1);
|
|
598
|
+
expect(new Set([...firstKeys, ...secondKeys]).size).toBe(10_001);
|
|
599
|
+
expect(secondKeys.some((key) => firstKeys.includes(key))).toBe(false);
|
|
600
|
+
expect(Object.values({ ...firstCheckpoint.upserts, ...secondCheckpoint.upserts }).every((entry) => entry.hash === "5633d479dfae75ba7a78914ee380fa202bd6126e7c6b7c22e3ebc9e1a6ddc871" &&
|
|
601
|
+
entry.size === 17 && entry.direction === "down" && entry.kind === "file")).toBe(true);
|
|
602
|
+
expect(firstCheckpoint).toMatchObject({
|
|
603
|
+
deletes: [],
|
|
604
|
+
metadata: { lastSync: "2026-09-11T12:00:00.000Z", pulls: [] },
|
|
605
|
+
});
|
|
606
|
+
expect(secondCheckpoint).toEqual({
|
|
607
|
+
upserts: expect.any(Object),
|
|
608
|
+
deletes: [],
|
|
609
|
+
});
|
|
610
|
+
expect(finalMetadata).toMatchObject({
|
|
611
|
+
upserts: {},
|
|
612
|
+
deletes: [],
|
|
613
|
+
metadata: {
|
|
614
|
+
lastSync: "2026-09-11T12:00:00.000Z",
|
|
615
|
+
pulls: [expect.objectContaining({
|
|
616
|
+
companyUid: "cmp_01ABCDEF",
|
|
617
|
+
startedAt: "2026-09-11T12:00:00.000Z",
|
|
618
|
+
completedAt: "2026-09-11T12:00:00.000Z",
|
|
619
|
+
})],
|
|
620
|
+
},
|
|
621
|
+
});
|
|
622
|
+
}, 60_000);
|
|
623
|
+
it("retains targeted-pull baselines as row fingerprints rather than entry objects", async () => {
|
|
624
|
+
writeJournal("acme", {
|
|
625
|
+
version: "2",
|
|
626
|
+
lastSync: "",
|
|
627
|
+
pulls: [],
|
|
628
|
+
files: {
|
|
629
|
+
"docs/known.md": {
|
|
630
|
+
hash: "a".repeat(64),
|
|
631
|
+
size: 1,
|
|
632
|
+
syncedAt: "2026-09-10T00:00:00.000Z",
|
|
633
|
+
direction: "down",
|
|
634
|
+
},
|
|
635
|
+
},
|
|
636
|
+
});
|
|
637
|
+
vi.mocked(remotePullModule.listRemoteForScope).mockResolvedValue([]);
|
|
638
|
+
const baselineBuilder = vi.spyOn(journalModule, "buildJournalBaseline");
|
|
639
|
+
let initialBaseline;
|
|
640
|
+
try {
|
|
641
|
+
await sync({
|
|
642
|
+
company: "acme",
|
|
643
|
+
vaultConfig: mockConfig,
|
|
644
|
+
hqRoot: tmpDir,
|
|
645
|
+
pullPrefixes: ["docs"],
|
|
646
|
+
skipReindex: true,
|
|
647
|
+
onEvent: () => undefined,
|
|
648
|
+
});
|
|
649
|
+
initialBaseline = baselineBuilder.mock.results[0]?.value;
|
|
650
|
+
}
|
|
651
|
+
finally {
|
|
652
|
+
baselineBuilder.mockRestore();
|
|
653
|
+
}
|
|
654
|
+
expect(initialBaseline).toBeDefined();
|
|
655
|
+
expect(initialBaseline.rows.get("docs/known.md")).toBeTypeOf("number");
|
|
656
|
+
expect(initialBaseline).not.toHaveProperty("files");
|
|
657
|
+
});
|
|
658
|
+
it("treats a key-order-only targeted journal row change as unchanged", () => {
|
|
659
|
+
const helpers = journalModule;
|
|
660
|
+
const initial = {
|
|
661
|
+
version: "2",
|
|
662
|
+
lastSync: "",
|
|
663
|
+
pulls: [],
|
|
664
|
+
files: {
|
|
665
|
+
"docs/key-order.md": {
|
|
666
|
+
hash: "a".repeat(64),
|
|
667
|
+
size: 1,
|
|
668
|
+
syncedAt: "2026-09-10T00:00:00.000Z",
|
|
669
|
+
direction: "down",
|
|
670
|
+
},
|
|
671
|
+
},
|
|
672
|
+
};
|
|
673
|
+
const reordered = {
|
|
674
|
+
...initial,
|
|
675
|
+
files: {
|
|
676
|
+
"docs/key-order.md": {
|
|
677
|
+
direction: "down",
|
|
678
|
+
syncedAt: "2026-09-10T00:00:00.000Z",
|
|
679
|
+
size: 1,
|
|
680
|
+
hash: "a".repeat(64),
|
|
681
|
+
},
|
|
682
|
+
},
|
|
683
|
+
};
|
|
684
|
+
expect(JSON.stringify(reordered.files["docs/key-order.md"])).not.toBe(JSON.stringify(initial.files["docs/key-order.md"]));
|
|
685
|
+
expect(helpers.journalDeltaFromBaseline(helpers.buildJournalBaseline(initial), reordered)).toMatchObject({ upserts: {}, deletes: [] });
|
|
686
|
+
});
|
|
423
687
|
it("receiver-sized targeted pulls retain exact LIST scope and materialize remote-new personal and continuity files", async () => {
|
|
424
688
|
// A receiver batch carries exact changed paths. The two real production
|
|
425
689
|
// shapes below exercise both a normal personal-vault file and the
|