@indigoai-us/hq-cloud 6.16.35 → 6.16.37
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin/sync-runner-watch-loop.test.js +1 -2
- package/dist/bin/sync-runner-watch-loop.test.js.map +1 -1
- package/dist/bin/sync-runner.test.js +6 -24
- package/dist/bin/sync-runner.test.js.map +1 -1
- package/dist/cli/sync.d.ts.map +1 -1
- package/dist/cli/sync.js +8 -19
- package/dist/cli/sync.js.map +1 -1
- package/dist/cli/sync.test.js +265 -1
- package/dist/cli/sync.test.js.map +1 -1
- package/dist/company-vault-transport.d.ts +0 -2
- package/dist/company-vault-transport.d.ts.map +1 -1
- package/dist/company-vault-transport.js +0 -2
- package/dist/company-vault-transport.js.map +1 -1
- package/dist/index.d.ts +2 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -1
- package/dist/index.js.map +1 -1
- package/dist/journal-row-store.d.ts +54 -2
- package/dist/journal-row-store.d.ts.map +1 -1
- package/dist/journal-row-store.js +305 -11
- package/dist/journal-row-store.js.map +1 -1
- package/dist/journal-snapshot-decoder.d.ts +15 -0
- package/dist/journal-snapshot-decoder.d.ts.map +1 -0
- package/dist/journal-snapshot-decoder.js +371 -0
- package/dist/journal-snapshot-decoder.js.map +1 -0
- package/dist/journal-snapshot-memory-helpers.d.ts +30 -0
- package/dist/journal-snapshot-memory-helpers.d.ts.map +1 -0
- package/dist/journal-snapshot-memory-helpers.js +212 -0
- package/dist/journal-snapshot-memory-helpers.js.map +1 -0
- package/dist/journal-snapshot-memory.test.d.ts +2 -0
- package/dist/journal-snapshot-memory.test.d.ts.map +1 -0
- package/dist/journal-snapshot-memory.test.js +32 -0
- package/dist/journal-snapshot-memory.test.js.map +1 -0
- package/dist/journal-snapshot-rows.d.ts +7 -0
- package/dist/journal-snapshot-rows.d.ts.map +1 -0
- package/dist/journal-snapshot-rows.js +10 -0
- package/dist/journal-snapshot-rows.js.map +1 -0
- package/dist/journal-snapshot-v4-memory.test.d.ts +2 -0
- package/dist/journal-snapshot-v4-memory.test.d.ts.map +1 -0
- package/dist/journal-snapshot-v4-memory.test.js +24 -0
- package/dist/journal-snapshot-v4-memory.test.js.map +1 -0
- package/dist/journal-snapshot-v4-writer-memory.test.d.ts +2 -0
- package/dist/journal-snapshot-v4-writer-memory.test.d.ts.map +1 -0
- package/dist/journal-snapshot-v4-writer-memory.test.js +76 -0
- package/dist/journal-snapshot-v4-writer-memory.test.js.map +1 -0
- package/dist/journal-snapshot-v4-writer.test.d.ts +2 -0
- package/dist/journal-snapshot-v4-writer.test.d.ts.map +1 -0
- package/dist/journal-snapshot-v4-writer.test.js +253 -0
- package/dist/journal-snapshot-v4-writer.test.js.map +1 -0
- package/dist/journal-snapshot-v4.d.ts +54 -0
- package/dist/journal-snapshot-v4.d.ts.map +1 -0
- package/dist/journal-snapshot-v4.js +386 -0
- package/dist/journal-snapshot-v4.js.map +1 -0
- package/dist/journal-snapshot-v4.test.d.ts +2 -0
- package/dist/journal-snapshot-v4.test.d.ts.map +1 -0
- package/dist/journal-snapshot-v4.test.js +280 -0
- package/dist/journal-snapshot-v4.test.js.map +1 -0
- package/dist/journal-store.test.js +163 -2
- package/dist/journal-store.test.js.map +1 -1
- package/dist/journal.d.ts +26 -2
- package/dist/journal.d.ts.map +1 -1
- package/dist/journal.js +64 -6
- package/dist/journal.js.map +1 -1
- package/dist/journal.test.js +57 -49
- package/dist/journal.test.js.map +1 -1
- package/dist/manifest/build-manifest.bench.test.js +165 -58
- package/dist/manifest/build-manifest.bench.test.js.map +1 -1
- package/dist/skill-telemetry.d.ts +1 -2
- package/dist/skill-telemetry.d.ts.map +1 -1
- package/dist/skill-telemetry.js +1 -10
- package/dist/skill-telemetry.js.map +1 -1
- package/dist/sync/child-process-sync-worker-env.test.d.ts +2 -0
- package/dist/sync/child-process-sync-worker-env.test.d.ts.map +1 -0
- package/dist/sync/child-process-sync-worker-env.test.js +38 -0
- package/dist/sync/child-process-sync-worker-env.test.js.map +1 -0
- package/dist/sync/feature-flags.d.ts +0 -16
- package/dist/sync/feature-flags.d.ts.map +1 -1
- package/dist/sync/feature-flags.js +0 -11
- package/dist/sync/feature-flags.js.map +1 -1
- package/dist/sync/index.d.ts +2 -2
- package/dist/sync/index.d.ts.map +1 -1
- package/dist/sync/index.js +1 -1
- package/dist/sync/index.js.map +1 -1
- package/dist/sync/logger.test.js +1 -3
- package/dist/sync/logger.test.js.map +1 -1
- package/dist/sync/metrics.test.js +0 -3
- package/dist/sync/metrics.test.js.map +1 -1
- package/dist/sync/push-receiver.d.ts +0 -7
- package/dist/sync/push-receiver.d.ts.map +1 -1
- package/dist/sync/push-receiver.js.map +1 -1
- package/dist/sync/state-store.d.ts +43 -1
- package/dist/sync/state-store.d.ts.map +1 -1
- package/dist/sync/state-store.js +132 -55
- package/dist/sync/state-store.js.map +1 -1
- package/dist/sync/state-store.test.js +82 -0
- package/dist/sync/state-store.test.js.map +1 -1
- package/dist/watcher.test.js +7 -8
- package/dist/watcher.test.js.map +1 -1
- package/package.json +1 -1
- package/dist/daemon-worker.d.ts +0 -11
- package/dist/daemon-worker.d.ts.map +0 -1
- package/dist/daemon-worker.js +0 -22
- package/dist/daemon-worker.js.map +0 -1
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"state-store.d.ts","sourceRoot":"","sources":["../../src/sync/state-store.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"state-store.d.ts","sourceRoot":"","sources":["../../src/sync/state-store.ts"],"names":[],"mappings":"AAWA,wEAAwE;AACxE,eAAO,MAAM,iBAAiB,qBAAoC,CAAC;AACnE,sFAAsF;AACtF,eAAO,MAAM,iBAAiB,qBAAoC,CAAC;AAmFnE,wBAAgB,2CAA2C,IAAI,IAAI,CAElE;AAED,wBAAgB,yCAAyC,IAAI,MAAM,CAElE;AAED,wBAAgB,kDAAkD,IAAI,IAAI,CAEzE;AAED,wBAAgB,gDAAgD,IAAI,MAAM,CAEzE;AAED,wBAAgB,yCAAyC,IAAI,IAAI,CAEhE;AAED,wBAAgB,uCAAuC,IAAI,MAAM,CAEhE;AAED,uFAAuF;AACvF,wBAAgB,2BAA2B,CACzC,UAAU,EAAE,MAAM,EAClB,WAAW,EAAE,MAAM,EACnB,OAAO,EAAE,OAAO,GACf;IAAE,MAAM,EAAE,SAAS,MAAM,EAAE,CAAC;IAAC,UAAU,EAAE,MAAM,CAAA;CAAE,CAEnD;AAED,yFAAyF;AACzF,wBAAgB,0BAA0B,CAAC,UAAU,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,GAAG,MAAM,CAE5G;AAED,6EAA6E;AAC7E,wBAAgB,qCAAqC,CAAC,IAAI,EAAE,CAAC,MAAM,IAAI,CAAC,GAAG,SAAS,GAAG,IAAI,CAE1F;AAED,kFAAkF;AAClF,wBAAgB,oCAAoC,CAAC,IAAI,EAAE,CAAC,MAAM,IAAI,CAAC,GAAG,SAAS,GAAG,IAAI,CAEzF;AAED,6FAA6F;AAC7F,wBAAgB,4CAA4C,CAAC,IAAI,EAAE,CAAC,MAAM,IAAI,CAAC,GAAG,SAAS,GAAG,IAAI,CAEjG;AAED,2EAA2E;AAC3E,wBAAgB,+BAA+B,CAAC,IAAI,EAAE,CAAC,MAAM,IAAI,CAAC,GAAG,SAAS,GAAG,IAAI,CAEpF;AAED,oFAAoF;AACpF,wBAAgB,4CAA4C,CAAC,IAAI,EAAE,CAAC,MAAM,IAAI,CAAC,GAAG,SAAS,GAAG,IAAI,CAEjG;AAED,0EAA0E;AAC1E,wBAAgB,mCAAmC,CAAC,IAAI,EAAE,CAAC,MAAM,IAAI,CAAC,GAAG,SAAS,GAAG,IAAI,CAExF;AAED,2DAA2D;AAC3D,wBAAgB,mCAAmC,CACjD,KAAK,EAAE,CAAC,CAAC,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,KAAK,MAAM,CAAC,GAAG,SAAS,GAC3G,IAAI,CAEN;AAED,4DAA4D;AAC5D,wBAAgB,0CAA0C,CAAC,IAAI,EAAE,CAAC,MAAM,IAAI,CAAC,GAAG,SAAS,GAAG,IAAI,CAE/F;AAED,+EAA+E;AAC/E,wBAAgB,4CAA4C,CAAC,IAAI,EAAE,CAAC,MAAM,IAAI,CAAC,GAAG,SAAS,GAAG,IAAI,CAEjG;AAED,sFAAsF;AACtF,wBAAgB,yCAAyC,CACvD,IAAI,EAAE,CAAC,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,GAAG,SAAS,GAC7C,IAAI,CAEN;AAED,+EAA+E;AAC/E,wBAAgB,+CAA+C,CAC7D,IAAI,EAAE,CAAC,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,GAAG,SAAS,GAC7C,IAAI,CAEN;AAED,6EAA6E;AAC7E,wBAAgB,sCAAsC,CAAC,IAAI,EAAE,CAAC,MAAM,IAAI,CAAC,GAAG,SAAS,GAAG,IAAI,CAE3F;AAED,qFAAqF;AACrF,wBAAgB,oCAAoC,CAAC,SAAS,EAAE,MAAM,GAAG,SAAS,GAAG,IAAI,CAExF;AAED,4FAA4F;AAC5F,wBAAgB,mCAAmC,CAAC,SAAS,EAAE,MAAM,GAAG,SAAS,GAAG,IAAI,CAEvF;AAED,gFAAgF;AAChF,wBAAgB,yBAAyB,CAAC,KAAK,EAAE,CAAC,MAAM,MAAM,CAAC,GAAG,SAAS,GAAG,IAAI,CAEjF;AAED,kFAAkF;AAClF,wBAAgB,yBAAyB,CAAC,OAAO,EAAE,CAAC,CAAC,EAAE,EAAE,MAAM,KAAK,IAAI,CAAC,GAAG,SAAS,GAAG,IAAI,CAE3F;AAED,4EAA4E;AAC5E,wBAAgB,0BAA0B,CAAC,MAAM,EAAE,CAAC,MAAM,MAAM,CAAC,GAAG,SAAS,GAAG,IAAI,CAEnF;AA+ED,MAAM,WAAW,gBAAgB,CAAC,CAAC,GAAG,OAAO;IAC3C,QAAQ,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,CAAC,CAAC;CACZ;AAED,MAAM,WAAW,iBAAiB,CAAC,CAAC;IAClC,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,YAAY,EAAE,CAAC,CAAC;IAChB,MAAM,EAAE,CAAC,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,gBAAgB,KAAK,CAAC,CAAC;IAClD;;;;OAIG;IACH,eAAe,CAAC,EAAE,yBAAyB,CAAC;IAC5C;;;;;OAKG;IACH,eAAe,CAAC,EAAE,yBAAyB,CAAC,CAAC,CAAC,CAAC;IAC/C;;;OAGG;IACH,cAAc,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IACnC;;;;OAIG;IACH,cAAc,CAAC,EAAE,CAAC,MAAM,EAAE,gBAAgB,KAAK,IAAI,CAAC;IACpD,4DAA4D;IAC5D,0BAA0B,CAAC,EAAE,MAAM,CAAC;IACpC,gFAAgF;IAChF,2BAA2B,CAAC,EAAE,MAAM,CAAC;CACtC;AAED,iFAAiF;AACjF,MAAM,WAAW,wBAAyB,SAAQ,QAAQ,CAAC,MAAM,CAAC;IAChE,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;CAC7B;AAED,kEAAkE;AAClE,MAAM,WAAW,yBAAyB,CAAC,CAAC;IAC1C,QAAQ,CAAC,kBAAkB,EAAE,MAAM,CAAC;IACpC,QAAQ,CAAC,KAAK,EAAE,CAAC,CAAC;CACnB;AAED,gEAAgE;AAChE,MAAM,WAAW,0BAA0B;IACzC,QAAQ,CAAC,MAAM,EAAE,SAAS,MAAM,EAAE,CAAC;IACnC,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;CAC7B;AAED,qFAAqF;AACrF,MAAM,MAAM,yBAAyB,GAAG,CAAC,MAAM,EAAE,wBAAwB,EAAE,KAAK,EAAE,MAAM,KAAK,OAAO,CAAC;AAErG,oFAAoF;AACpF,MAAM,MAAM,yBAAyB,CAAC,CAAC,IAAI,CACzC,UAAU,EAAE,MAAM,EAClB,WAAW,EAAE,MAAM,EACnB,OAAO,EAAE,yBAAyB,CAAC,CAAC,CAAC,KAClC,0BAA0B,CAAC;AAEhC,MAAM,WAAW,uBAAuB;IACtC,iEAAiE;IACjE,OAAO,EAAE,gBAAgB,EAAE,CAAC;IAC5B,4EAA4E;IAC5E,SAAS,EAAE,OAAO,CAAC;CACpB;AAED,MAAM,WAAW,sBAAuB,SAAQ,gBAAgB;IAC9D,OAAO,EAAE,uBAAuB,CAAC;CAClC;AAED;;;;;;;;GAQG;AACH,MAAM,WAAW,qBAAqB,CAAC,CAAC;IACtC,sEAAsE;IACtE,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,UAAU,CAAC,CAAC,CAAC,CAAC;IACrB,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,OAAO,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,uBAAuB;IACtC,oEAAoE;IACpE,QAAQ,EAAE,MAAM,CAAC;IACjB,6EAA6E;IAC7E,YAAY,EAAE,MAAM,CAAC;CACtB;AAED,kEAAkE;AAClE,MAAM,WAAW,4BAA4B;IAC3C,UAAU,EAAE,MAAM,CAAC;IACnB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED;;;;;;GAMG;AACH,MAAM,WAAW,uBAAuB;IACtC,QAAQ,CAAC,QAAQ,EAAE,4BAA4B,CAAC;IAChD,2EAA2E;IAC3E,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,OAAO,IAAI,IAAI,CAAC;CACjB;AAED,+DAA+D;AAC/D,MAAM,WAAW,oBAAoB;IACnC,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,OAAO,CAAC;CAChB;AAED,MAAM,WAAW,sBAAsB;IACrC,mBAAmB,EAAE,MAAM,CAAC;IAC5B,iBAAiB,EAAE,MAAM,CAAC;IAC1B,sBAAsB,EAAE,MAAM,CAAC;IAC/B,iBAAiB,EAAE,MAAM,CAAC;CAC3B;AAED,MAAM,WAAW,0BAA0B;IACzC,uEAAuE;IACvE,SAAS,EAAE,MAAM,CAAC;IAClB,mDAAmD;IACnD,UAAU,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,MAAM,2BAA2B,GACnC;IAAE,MAAM,EAAE,SAAS,GAAG,YAAY,CAAA;CAAE,GACpC,CAAC;IAAE,MAAM,EAAE,UAAU,CAAA;CAAE,GAAG,sBAAsB,CAAC,CAAC;AAEtD,MAAM,WAAW,4BAA4B;IAC3C,yEAAyE;IACzE,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,uEAAuE;IACvE,UAAU,CAAC,EAAE,CAAC,KAAK,EAAE,0BAA0B,KAAK,IAAI,CAAC;CAC1D;AAiCD,qBAAa,yBAA0B,SAAQ,KAAK;gBACtC,OAAO,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,YAAY;CAIpD;AAED,MAAM,MAAM,yBAAyB,GAAG,yBAAyB,CAAC;AAElE,qBAAa,mBAAoB,SAAQ,KAAK;IAC5C,QAAQ,CAAC,MAAM,CAAC,EAAE,yBAAyB,CAAC;gBAEhC,OAAO,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,YAAY,GAAG;QAAE,MAAM,CAAC,EAAE,yBAAyB,CAAA;KAAE;CAK7F;AAYD,+DAA+D;AAC/D,qBAAa,UAAU,CAAC,CAAC;IAarB,OAAO,CAAC,QAAQ,CAAC,OAAO;IAZ1B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,UAAU,EAAE,OAAO,CAAC;IAC7B,OAAO,CAAC,UAAU,CAAS;IAC3B,OAAO,CAAC,KAAK,CAAI;IACjB,OAAO,CAAC,kBAAkB,CAAS;IACnC,OAAO,CAAC,WAAW,CAAS;IAC5B,OAAO,CAAC,QAAQ,CAAS;IACzB,OAAO,CAAC,oBAAoB,CAAS;IACrC,OAAO,CAAC,iCAAiC,CAAS;IAElD,OAAO;IAiBP,MAAM,CAAC,MAAM,CAAC,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,OAAO;IAUxD,MAAM,CAAC,IAAI,CAAC,CAAC,EAAE,OAAO,EAAE,iBAAiB,CAAC,CAAC,CAAC,GAAG,UAAU,CAAC,CAAC,CAAC;IA6C5D,uEAAuE;WAC1D,SAAS,CAAC,CAAC,EAAE,OAAO,EAAE,iBAAiB,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;IA4ChF;;;;;;;;;OASG;IACH,MAAM,CAAC,WAAW,CAAC,CAAC,EAAE,OAAO,EAAE,SAAS,qBAAqB,CAAC,CAAC,CAAC,EAAE,GAAG,sBAAsB,EAAE;IA2D7F;;;;;OAKG;IACH,MAAM,CAAC,cAAc,CAAC,CAAC,EACrB,OAAO,EAAE,iBAAiB,CAAC,CAAC,CAAC,EAC7B,WAAW,GAAE,4BAAiC,GAC7C,2BAA2B;IA+G9B,QAAQ,IAAI,CAAC;IAIb,aAAa,IAAI,MAAM;IAIvB,qBAAqB,IAAI,MAAM;IAI/B,gBAAgB,IAAI,uBAAuB;IAI3C;;;;;;OAMG;IACH,cAAc,CACZ,WAAW,GAAE,4BAAiC,GAC7C,2BAA2B;IAI9B,8EAA8E;IAC9E,eAAe,IAAI,OAAO;IAO1B,oEAAoE;YACtD,oBAAoB;IAiFlC,qBAAqB,IAAI,4BAA4B;IAQrD;;;;;;;OAOG;IACH,gBAAgB,CAAC,QAAQ,GAAE,4BAA2D,GAAG,OAAO;IAUhG;;;;OAIG;IACH,oBAAoB,IAAI,uBAAuB;IA4B/C;;;;OAIG;IACF,mBAAmB,CAAC,IAAI,EAAE,SAAS,MAAM,EAAE,GAAG,gBAAgB,CAAC,oBAAoB,CAAC;IAgBrF,MAAM,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,KAAK,SAAI,GAAG,sBAAsB;IA4BzE;;;;OAIG;IACG,WAAW,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,KAAK,SAAI,GAAG,OAAO,CAAC,sBAAsB,CAAC;IAgC7F,iEAAiE;IAC3D,aAAa,CACjB,IAAI,EAAE,MAAM,EACZ,OAAO,EAAE,OAAO,EAChB,YAAY,EAAE,MAAM,OAAO,EAC3B,KAAK,SAAI,GACR,OAAO,CAAC,sBAAsB,GAAG,SAAS,CAAC;IAiC9C;;;;;;OAMG;IACG,sBAAsB,CAC1B,IAAI,EAAE,MAAM,EACZ,cAAc,EAAE,MAAM,OAAO,GAAG,SAAS,EACzC,KAAK,SAAI,GACR,OAAO,CAAC,sBAAsB,GAAG,SAAS,CAAC;IAkC9C,sEAAsE;IACtE,OAAO,IAAI,uBAAuB;IAUlC,6DAA6D;IACvD,YAAY,IAAI,OAAO,CAAC,uBAAuB,CAAC;IAUtD,OAAO,CAAC,cAAc;IAiCtB;;;;;;;;OAQG;IACH,YAAY,CAAC,SAAS,EAAE,CAAC,GAAG,IAAI;IAIhC;;;;;;;OAOG;IACH,gBAAgB,CAAC,SAAS,EAAE,CAAC,KAAK,EAAE,CAAC,KAAK,CAAC,GAAG,IAAI;IAWlD,OAAO,CAAC,8BAA8B;IAWtC,OAAO,CAAC,mBAAmB;IAgE3B,uEAAuE;IACvE,OAAO,IAAI,IAAI;IAuBf,OAAO,CAAC,eAAe;IA+BvB,OAAO,CAAC,kCAAkC;YAY5B,uCAAuC;IAYrD,OAAO,CAAC,eAAe;IAKvB,OAAO,CAAC,cAAc;IAqBtB,gFAAgF;IAChF,OAAO,CAAC,qBAAqB;IAsB7B,OAAO,CAAC,iBAAiB;CAS1B"}
|
package/dist/sync/state-store.js
CHANGED
|
@@ -8,7 +8,10 @@
|
|
|
8
8
|
import * as crypto from "node:crypto";
|
|
9
9
|
import * as fs from "node:fs";
|
|
10
10
|
import * as path from "node:path";
|
|
11
|
-
|
|
11
|
+
/** The legacy JSON snapshot format emitted by this release's writer. */
|
|
12
|
+
export const SNAPSHOT_V3_MAGIC = Buffer.from("HQSNAP3\0", "ascii");
|
|
13
|
+
/** The packed journal snapshot format accepted by the reader before it is emitted. */
|
|
14
|
+
export const SNAPSHOT_V4_MAGIC = Buffer.from("HQSNAP4\0", "ascii");
|
|
12
15
|
const WAL_MAGIC = Buffer.from("HQWAL3\0\0", "ascii");
|
|
13
16
|
const FRAME_MAGIC = 0x48515733;
|
|
14
17
|
const SNAPSHOT_HEADER_BYTES = 8 + 8 + 4 + 32 + 32;
|
|
@@ -74,6 +77,7 @@ let snapshotMaterializationCount = 0;
|
|
|
74
77
|
let snapshotEncodeCount = 0;
|
|
75
78
|
let snapshotWriteHookForTest;
|
|
76
79
|
let snapshotReadHookForTest;
|
|
80
|
+
let snapshotMemorySampleHookForTest;
|
|
77
81
|
let walReadHookForTest;
|
|
78
82
|
let afterGenerationStageHookForTest;
|
|
79
83
|
let reflinkCopyHookForTest;
|
|
@@ -122,6 +126,10 @@ export function setStateStoreSnapshotWriteHookForTest(hook) {
|
|
|
122
126
|
export function setStateStoreSnapshotReadHookForTest(hook) {
|
|
123
127
|
snapshotReadHookForTest = hook;
|
|
124
128
|
}
|
|
129
|
+
/** Test-only sampling seam before and after decoder work for each bounded snapshot chunk. */
|
|
130
|
+
export function setStateStoreSnapshotMemorySampleHookForTest(hook) {
|
|
131
|
+
snapshotMemorySampleHookForTest = hook;
|
|
132
|
+
}
|
|
125
133
|
/** Test-only seam after the WAL length is sampled during cold recovery. */
|
|
126
134
|
export function setStateStoreWalReadHookForTest(hook) {
|
|
127
135
|
walReadHookForTest = hook;
|
|
@@ -297,7 +305,7 @@ export class StateStore {
|
|
|
297
305
|
const dir = stateDirectory(normalized.rootDir, scopeDigest);
|
|
298
306
|
fs.mkdirSync(dir, { recursive: true });
|
|
299
307
|
while (true) {
|
|
300
|
-
const recovered = readNewestGenerationOptimistically(dir, scopeDigest, normalized.reduce);
|
|
308
|
+
const recovered = readNewestGenerationOptimistically(dir, scopeDigest, normalized.reduce, normalized.snapshotDecoder, normalized.snapshotMagics);
|
|
301
309
|
if (recovered) {
|
|
302
310
|
const store = new StateStore(normalized, recovered, false);
|
|
303
311
|
// A process can die after its durable append but before append() gets
|
|
@@ -322,7 +330,7 @@ export class StateStore {
|
|
|
322
330
|
walBytes: WAL_HEADER_BYTES,
|
|
323
331
|
maxReadBytes: 0,
|
|
324
332
|
};
|
|
325
|
-
writeGeneration(dir, scopeDigest, initial);
|
|
333
|
+
writeGeneration(dir, scopeDigest, initial, normalized.snapshotEncoder);
|
|
326
334
|
writeCurrentPointer(dir, initial.generation);
|
|
327
335
|
return new StateStore(normalized, initial, true);
|
|
328
336
|
});
|
|
@@ -337,7 +345,7 @@ export class StateStore {
|
|
|
337
345
|
const dir = stateDirectory(normalized.rootDir, scopeDigest);
|
|
338
346
|
await fs.promises.mkdir(dir, { recursive: true });
|
|
339
347
|
while (true) {
|
|
340
|
-
const recovered = await readNewestGenerationOptimisticallyAsync(dir, scopeDigest, normalized.reduce);
|
|
348
|
+
const recovered = await readNewestGenerationOptimisticallyAsync(dir, scopeDigest, normalized.reduce, normalized.snapshotDecoder, normalized.snapshotMagics);
|
|
341
349
|
if (recovered) {
|
|
342
350
|
const store = new StateStore(normalized, recovered, false);
|
|
343
351
|
// Match open(): a recovered over-budget WAL is compacted before this
|
|
@@ -361,7 +369,7 @@ export class StateStore {
|
|
|
361
369
|
walBytes: WAL_HEADER_BYTES,
|
|
362
370
|
maxReadBytes: 0,
|
|
363
371
|
};
|
|
364
|
-
writeGeneration(dir, scopeDigest, initial);
|
|
372
|
+
writeGeneration(dir, scopeDigest, initial, normalized.snapshotEncoder);
|
|
365
373
|
writeCurrentPointer(dir, initial.generation);
|
|
366
374
|
return new StateStore(normalized, initial, true);
|
|
367
375
|
});
|
|
@@ -471,7 +479,7 @@ export class StateStore {
|
|
|
471
479
|
// Preserve recovery's corruption classification for a crashed or
|
|
472
480
|
// malformed store that has snapshots but no complete generation.
|
|
473
481
|
if (hasSnapshotGeneration(dir)) {
|
|
474
|
-
readNewestGeneration(dir, scopeDigest, normalized.reduce, maintenance.onProgress, false);
|
|
482
|
+
readNewestGeneration(dir, scopeDigest, normalized.reduce, normalized.snapshotDecoder, normalized.snapshotMagics, maintenance.onProgress, false);
|
|
475
483
|
continue;
|
|
476
484
|
}
|
|
477
485
|
return { status: "missing" };
|
|
@@ -481,7 +489,7 @@ export class StateStore {
|
|
|
481
489
|
// Snapshot and WAL replay are read-only. Do not make appenders wait for
|
|
482
490
|
// this potentially multi-second materialisation; validate its view before
|
|
483
491
|
// using it to build a replacement generation.
|
|
484
|
-
const recovered = readNewestGeneration(dir, scopeDigest, normalized.reduce, maintenance.onProgress, false);
|
|
492
|
+
const recovered = readNewestGeneration(dir, scopeDigest, normalized.reduce, normalized.snapshotDecoder, normalized.snapshotMagics, maintenance.onProgress, false);
|
|
485
493
|
if (!recovered)
|
|
486
494
|
continue;
|
|
487
495
|
plan = withScopeLock(dir, () => {
|
|
@@ -505,7 +513,7 @@ export class StateStore {
|
|
|
505
513
|
// from a copy-on-write clone of the first so one logical repair has one
|
|
506
514
|
// full snapshot materialisation on filesystems that support reflinks.
|
|
507
515
|
const next = { ...first, generation: first.generation + 1 };
|
|
508
|
-
const stagedFirst = stageGeneration(dir, scopeDigest, first);
|
|
516
|
+
const stagedFirst = stageGeneration(dir, scopeDigest, first, normalized.snapshotEncoder);
|
|
509
517
|
let staged;
|
|
510
518
|
try {
|
|
511
519
|
staged = stageGenerationFromSnapshot(dir, scopeDigest, stagedFirst.snapshotTemporary, next);
|
|
@@ -633,7 +641,7 @@ export class StateStore {
|
|
|
633
641
|
return false;
|
|
634
642
|
attempts += 1;
|
|
635
643
|
try {
|
|
636
|
-
const staged = stageGeneration(this.stateDir, this.scopeDigest, plan.next);
|
|
644
|
+
const staged = stageGeneration(this.stateDir, this.scopeDigest, plan.next, this.options.snapshotEncoder);
|
|
637
645
|
try {
|
|
638
646
|
afterGenerationStageHookForTest?.();
|
|
639
647
|
}
|
|
@@ -979,7 +987,7 @@ export class StateStore {
|
|
|
979
987
|
this.recoverNewestGenerationOutsideLock();
|
|
980
988
|
return false;
|
|
981
989
|
}
|
|
982
|
-
const first = stageGeneration(this.stateDir, this.scopeDigest, plan.first);
|
|
990
|
+
const first = stageGeneration(this.stateDir, this.scopeDigest, plan.first, this.options.snapshotEncoder);
|
|
983
991
|
let second;
|
|
984
992
|
try {
|
|
985
993
|
second = plan.current.generation === plan.first.generation
|
|
@@ -1058,13 +1066,13 @@ export class StateStore {
|
|
|
1058
1066
|
return undefined;
|
|
1059
1067
|
}
|
|
1060
1068
|
recoverNewestGenerationOutsideLock() {
|
|
1061
|
-
const recovered = readNewestGenerationOptimistically(this.stateDir, this.scopeDigest, this.options.reduce);
|
|
1069
|
+
const recovered = readNewestGenerationOptimistically(this.stateDir, this.scopeDigest, this.options.reduce, this.options.snapshotDecoder, this.options.snapshotMagics);
|
|
1062
1070
|
if (!recovered)
|
|
1063
1071
|
throw new StateStoreCorruptionError("no valid generation remains while appending");
|
|
1064
1072
|
this.installGeneration(recovered);
|
|
1065
1073
|
}
|
|
1066
1074
|
async recoverNewestGenerationOutsideLockAsync() {
|
|
1067
|
-
const recovered = await readNewestGenerationOptimisticallyAsync(this.stateDir, this.scopeDigest, this.options.reduce);
|
|
1075
|
+
const recovered = await readNewestGenerationOptimisticallyAsync(this.stateDir, this.scopeDigest, this.options.reduce, this.options.snapshotDecoder, this.options.snapshotMagics);
|
|
1068
1076
|
if (!recovered)
|
|
1069
1077
|
throw new StateStoreCorruptionError("no valid generation remains while appending");
|
|
1070
1078
|
this.installGeneration(recovered);
|
|
@@ -1090,7 +1098,7 @@ export class StateStore {
|
|
|
1090
1098
|
}
|
|
1091
1099
|
/** Materialise off-lock, then publish only if no writer advanced the source. */
|
|
1092
1100
|
publishCompactionPlan(plan) {
|
|
1093
|
-
const staged = stageGeneration(this.stateDir, this.scopeDigest, plan.next);
|
|
1101
|
+
const staged = stageGeneration(this.stateDir, this.scopeDigest, plan.next, this.options.snapshotEncoder);
|
|
1094
1102
|
try {
|
|
1095
1103
|
afterGenerationStageHookForTest?.();
|
|
1096
1104
|
return withScopeLock(this.stateDir, () => {
|
|
@@ -1121,7 +1129,7 @@ export class StateStore {
|
|
|
1121
1129
|
this.observedHighestSnapshotGeneration = generation.observedHighestSnapshotGeneration ?? generation.generation;
|
|
1122
1130
|
}
|
|
1123
1131
|
}
|
|
1124
|
-
function readNewestGeneration(dir, scopeDigest, reduce, onProgress, repairTornTail = true) {
|
|
1132
|
+
function readNewestGeneration(dir, scopeDigest, reduce, snapshotDecoder, snapshotMagics, onProgress, repairTornTail = true) {
|
|
1125
1133
|
const names = fs.readdirSync(dir);
|
|
1126
1134
|
const generations = names
|
|
1127
1135
|
.map((name) => /^snapshot-(\d+)\.bin$/.exec(name)?.[1])
|
|
@@ -1134,7 +1142,7 @@ function readNewestGeneration(dir, scopeDigest, reduce, onProgress, repairTornTa
|
|
|
1134
1142
|
for (const generation of generations) {
|
|
1135
1143
|
try {
|
|
1136
1144
|
return {
|
|
1137
|
-
...readGeneration(dir, scopeDigest, generation, reduce, onProgress, repairTornTail),
|
|
1145
|
+
...readGeneration(dir, scopeDigest, generation, reduce, snapshotDecoder, snapshotMagics, onProgress, repairTornTail),
|
|
1138
1146
|
observedHighestSnapshotGeneration: generations[0],
|
|
1139
1147
|
};
|
|
1140
1148
|
}
|
|
@@ -1163,10 +1171,10 @@ function readNewestGeneration(dir, scopeDigest, reduce, onProgress, repairTornTa
|
|
|
1163
1171
|
}
|
|
1164
1172
|
return undefined;
|
|
1165
1173
|
}
|
|
1166
|
-
function readGeneration(dir, scopeDigest, generation, reduce, onProgress, repairTornTail = true) {
|
|
1174
|
+
function readGeneration(dir, scopeDigest, generation, reduce, snapshotDecoder, snapshotMagics, onProgress, repairTornTail = true) {
|
|
1167
1175
|
const snapshotBytes = generationFileSize(snapshotPath(dir, generation), "snapshot", generation);
|
|
1168
1176
|
const totalBytes = snapshotBytes + generationFileSize(walPath(dir, generation), "WAL", generation);
|
|
1169
|
-
const snapshot = readSnapshot(snapshotPath(dir, generation), scopeDigest, generation);
|
|
1177
|
+
const snapshot = readSnapshot(snapshotPath(dir, generation), scopeDigest, generation, snapshotDecoder, snapshotMagics);
|
|
1170
1178
|
onProgress?.({ bytesRead: snapshotBytes, totalBytes });
|
|
1171
1179
|
const parsed = readWal(walPath(dir, generation), scopeDigest, generation, snapshot.payload.lastRecordSequence, snapshot.payload.state, reduce, snapshotBytes, totalBytes, onProgress, repairTornTail);
|
|
1172
1180
|
return {
|
|
@@ -1187,7 +1195,7 @@ function readGeneration(dir, scopeDigest, generation, reduce, onProgress, repair
|
|
|
1187
1195
|
* WAL suffix rather than restarting the expensive immutable snapshot read.
|
|
1188
1196
|
* Rotation or a torn suffix remains lock-protected.
|
|
1189
1197
|
*/
|
|
1190
|
-
function readNewestGenerationOptimistically(dir, scopeDigest, reduce, onProgress) {
|
|
1198
|
+
function readNewestGenerationOptimistically(dir, scopeDigest, reduce, snapshotDecoder, snapshotMagics, onProgress) {
|
|
1191
1199
|
let resamples = 0;
|
|
1192
1200
|
while (true) {
|
|
1193
1201
|
const candidate = withScopeLock(dir, () => {
|
|
@@ -1198,7 +1206,7 @@ function readNewestGenerationOptimistically(dir, scopeDigest, reduce, onProgress
|
|
|
1198
1206
|
return undefined;
|
|
1199
1207
|
let recovered;
|
|
1200
1208
|
try {
|
|
1201
|
-
recovered = readNewestGeneration(dir, scopeDigest, reduce, onProgress, false);
|
|
1209
|
+
recovered = readNewestGeneration(dir, scopeDigest, reduce, snapshotDecoder, snapshotMagics, onProgress, false);
|
|
1202
1210
|
}
|
|
1203
1211
|
catch (error) {
|
|
1204
1212
|
if (!(error instanceof StateStoreGenerationMissingError))
|
|
@@ -1218,7 +1226,7 @@ function readNewestGenerationOptimistically(dir, scopeDigest, reduce, onProgress
|
|
|
1218
1226
|
return validated;
|
|
1219
1227
|
}
|
|
1220
1228
|
}
|
|
1221
|
-
async function readNewestGenerationOptimisticallyAsync(dir, scopeDigest, reduce, onProgress) {
|
|
1229
|
+
async function readNewestGenerationOptimisticallyAsync(dir, scopeDigest, reduce, snapshotDecoder, snapshotMagics, onProgress) {
|
|
1222
1230
|
let resamples = 0;
|
|
1223
1231
|
while (true) {
|
|
1224
1232
|
const candidate = await withScopeLockAsync(dir, () => {
|
|
@@ -1229,7 +1237,7 @@ async function readNewestGenerationOptimisticallyAsync(dir, scopeDigest, reduce,
|
|
|
1229
1237
|
return undefined;
|
|
1230
1238
|
let recovered;
|
|
1231
1239
|
try {
|
|
1232
|
-
recovered = readNewestGeneration(dir, scopeDigest, reduce, onProgress, false);
|
|
1240
|
+
recovered = readNewestGeneration(dir, scopeDigest, reduce, snapshotDecoder, snapshotMagics, onProgress, false);
|
|
1233
1241
|
}
|
|
1234
1242
|
catch (error) {
|
|
1235
1243
|
if (!(error instanceof StateStoreGenerationMissingError))
|
|
@@ -1311,11 +1319,8 @@ function needsMaintenance(dir, newestGeneration, options) {
|
|
|
1311
1319
|
retainedGenerationHistoryNeedsRepair(dir, newestGeneration, currentGeneration) ||
|
|
1312
1320
|
walBytes > options.maxWalBytesBeforeCompaction;
|
|
1313
1321
|
}
|
|
1314
|
-
function writeGeneration(dir, scopeDigest, generation) {
|
|
1315
|
-
const snapshot =
|
|
1316
|
-
lastRecordSequence: generation.lastRecordSequence,
|
|
1317
|
-
state: generation.state,
|
|
1318
|
-
});
|
|
1322
|
+
function writeGeneration(dir, scopeDigest, generation, snapshotEncoder) {
|
|
1323
|
+
const snapshot = encodeGenerationSnapshot(generation, scopeDigest, snapshotEncoder);
|
|
1319
1324
|
const destination = snapshotPath(dir, generation.generation);
|
|
1320
1325
|
writeDurableTempThenRename(dir, destination, snapshot.chunks);
|
|
1321
1326
|
const header = encodeWalHeader(generation.generation, scopeDigest);
|
|
@@ -1326,12 +1331,9 @@ function writeGeneration(dir, scopeDigest, generation) {
|
|
|
1326
1331
|
* Temporary names never match the recovery generation pattern, so a process
|
|
1327
1332
|
* crash here leaves the prior complete generation authoritative.
|
|
1328
1333
|
*/
|
|
1329
|
-
function stageGeneration(dir, scopeDigest, generation) {
|
|
1334
|
+
function stageGeneration(dir, scopeDigest, generation, snapshotEncoder) {
|
|
1330
1335
|
snapshotMaterializationCount++;
|
|
1331
|
-
const snapshot =
|
|
1332
|
-
lastRecordSequence: generation.lastRecordSequence,
|
|
1333
|
-
state: generation.state,
|
|
1334
|
-
});
|
|
1336
|
+
const snapshot = encodeGenerationSnapshot(generation, scopeDigest, snapshotEncoder);
|
|
1335
1337
|
let snapshotTemporary;
|
|
1336
1338
|
let walTemporary;
|
|
1337
1339
|
try {
|
|
@@ -1353,6 +1355,14 @@ function stageGeneration(dir, scopeDigest, generation) {
|
|
|
1353
1355
|
throw error;
|
|
1354
1356
|
}
|
|
1355
1357
|
}
|
|
1358
|
+
function encodeGenerationSnapshot(generation, scopeDigest, snapshotEncoder) {
|
|
1359
|
+
const payload = {
|
|
1360
|
+
lastRecordSequence: generation.lastRecordSequence,
|
|
1361
|
+
state: generation.state,
|
|
1362
|
+
};
|
|
1363
|
+
return snapshotEncoder?.(generation.generation, scopeDigest, payload)
|
|
1364
|
+
?? encodeSnapshotChunks(generation.generation, scopeDigest, payload);
|
|
1365
|
+
}
|
|
1356
1366
|
/**
|
|
1357
1367
|
* Make the rollback copy without serialising or writing the snapshot payload a
|
|
1358
1368
|
* second time when the store directory is on a reflink-capable filesystem.
|
|
@@ -1445,7 +1455,85 @@ function discardStagedGeneration(staged) {
|
|
|
1445
1455
|
}
|
|
1446
1456
|
catch { /* staged files are best-effort cleanup */ }
|
|
1447
1457
|
}
|
|
1448
|
-
function
|
|
1458
|
+
function readSnapshotChecksum(fd, payloadOffset, payloadLength, header, generation) {
|
|
1459
|
+
const hash = crypto.createHash("sha256").update(header);
|
|
1460
|
+
let offset = 0;
|
|
1461
|
+
while (offset < payloadLength) {
|
|
1462
|
+
const length = Math.min(SNAPSHOT_CHUNK_BYTES, payloadLength - offset);
|
|
1463
|
+
const chunk = Buffer.allocUnsafe(length);
|
|
1464
|
+
if (readIntoSync(fd, chunk, payloadOffset + offset) !== length) {
|
|
1465
|
+
throw new StateStoreCorruptionError(`snapshot generation ${generation} has a torn payload`);
|
|
1466
|
+
}
|
|
1467
|
+
hash.update(chunk);
|
|
1468
|
+
snapshotMemorySampleHookForTest?.();
|
|
1469
|
+
offset += length;
|
|
1470
|
+
}
|
|
1471
|
+
const checksum = Buffer.alloc(32);
|
|
1472
|
+
if (readIntoSync(fd, checksum, payloadOffset + payloadLength) !== checksum.length) {
|
|
1473
|
+
throw new StateStoreCorruptionError(`snapshot generation ${generation} has a torn checksum`);
|
|
1474
|
+
}
|
|
1475
|
+
if (!hash.digest().equals(checksum)) {
|
|
1476
|
+
throw new StateStoreCorruptionError(`snapshot generation ${generation} has a bad checksum`);
|
|
1477
|
+
}
|
|
1478
|
+
}
|
|
1479
|
+
function readSnapshotPayloadChunks(fd, payloadOffset, payloadLength, generation) {
|
|
1480
|
+
return {
|
|
1481
|
+
byteLength: payloadLength,
|
|
1482
|
+
*[Symbol.iterator]() {
|
|
1483
|
+
let offset = 0;
|
|
1484
|
+
while (offset < payloadLength) {
|
|
1485
|
+
const length = Math.min(SNAPSHOT_CHUNK_BYTES, payloadLength - offset);
|
|
1486
|
+
const chunk = Buffer.allocUnsafe(length);
|
|
1487
|
+
if (readIntoSync(fd, chunk, payloadOffset + offset) !== length) {
|
|
1488
|
+
throw new StateStoreCorruptionError(`snapshot generation ${generation} has a torn payload`);
|
|
1489
|
+
}
|
|
1490
|
+
snapshotMemorySampleHookForTest?.();
|
|
1491
|
+
offset += length;
|
|
1492
|
+
yield chunk;
|
|
1493
|
+
snapshotMemorySampleHookForTest?.();
|
|
1494
|
+
}
|
|
1495
|
+
},
|
|
1496
|
+
};
|
|
1497
|
+
}
|
|
1498
|
+
function decodeSnapshotWholePayload(fd, header, payloadLength, generation) {
|
|
1499
|
+
const data = Buffer.allocUnsafe(payloadLength);
|
|
1500
|
+
if (readIntoSync(fd, data, SNAPSHOT_FIXED_HEADER_BYTES) !== payloadLength) {
|
|
1501
|
+
throw new StateStoreCorruptionError(`snapshot generation ${generation} has a torn payload`);
|
|
1502
|
+
}
|
|
1503
|
+
snapshotMemorySampleHookForTest?.();
|
|
1504
|
+
const checksum = Buffer.alloc(32);
|
|
1505
|
+
if (readIntoSync(fd, checksum, SNAPSHOT_FIXED_HEADER_BYTES + payloadLength) !== checksum.length) {
|
|
1506
|
+
throw new StateStoreCorruptionError(`snapshot generation ${generation} has a torn checksum`);
|
|
1507
|
+
}
|
|
1508
|
+
const expected = crypto.createHash("sha256").update(header).update(data).digest();
|
|
1509
|
+
if (!expected.equals(checksum)) {
|
|
1510
|
+
throw new StateStoreCorruptionError(`snapshot generation ${generation} has a bad checksum`);
|
|
1511
|
+
}
|
|
1512
|
+
try {
|
|
1513
|
+
const text = data.toString("utf8");
|
|
1514
|
+
snapshotMemorySampleHookForTest?.();
|
|
1515
|
+
const decoded = JSON.parse(text);
|
|
1516
|
+
snapshotMemorySampleHookForTest?.();
|
|
1517
|
+
return decoded;
|
|
1518
|
+
}
|
|
1519
|
+
catch (error) {
|
|
1520
|
+
throw new StateStoreCorruptionError(`snapshot generation ${generation} has invalid JSON`, { cause: error });
|
|
1521
|
+
}
|
|
1522
|
+
}
|
|
1523
|
+
function decodeSnapshotIncrementally(fd, header, payloadLength, generation, snapshotDecoder, magic) {
|
|
1524
|
+
readSnapshotChecksum(fd, SNAPSHOT_FIXED_HEADER_BYTES, payloadLength, header, generation);
|
|
1525
|
+
try {
|
|
1526
|
+
const decoded = snapshotDecoder(readSnapshotPayloadChunks(fd, SNAPSHOT_FIXED_HEADER_BYTES, payloadLength, generation), magic);
|
|
1527
|
+
snapshotMemorySampleHookForTest?.();
|
|
1528
|
+
return decoded;
|
|
1529
|
+
}
|
|
1530
|
+
catch (error) {
|
|
1531
|
+
if (error instanceof StateStoreCorruptionError)
|
|
1532
|
+
throw error;
|
|
1533
|
+
throw new StateStoreCorruptionError(`snapshot generation ${generation} has invalid JSON`, { cause: error });
|
|
1534
|
+
}
|
|
1535
|
+
}
|
|
1536
|
+
function readSnapshot(filePath, scopeDigest, generation, snapshotDecoder, snapshotMagics) {
|
|
1449
1537
|
snapshotRecoveryCount++;
|
|
1450
1538
|
let fd;
|
|
1451
1539
|
try {
|
|
@@ -1463,7 +1551,9 @@ function readSnapshot(filePath, scopeDigest, generation) {
|
|
|
1463
1551
|
if (readIntoSync(fd, header, 0) !== header.length) {
|
|
1464
1552
|
throw new StateStoreCorruptionError(`snapshot generation ${generation} has a torn header`);
|
|
1465
1553
|
}
|
|
1466
|
-
|
|
1554
|
+
const magic = header.subarray(0, 8);
|
|
1555
|
+
const acceptedMagics = [SNAPSHOT_V3_MAGIC, ...(snapshotMagics ?? [])];
|
|
1556
|
+
if (!acceptedMagics.some((accepted) => magic.equals(accepted))) {
|
|
1467
1557
|
throw new StateStoreCorruptionError(`snapshot generation ${generation} has an invalid magic`);
|
|
1468
1558
|
}
|
|
1469
1559
|
if (readU64(header, 8) !== generation) {
|
|
@@ -1477,29 +1567,16 @@ function readSnapshot(filePath, scopeDigest, generation) {
|
|
|
1477
1567
|
throw new StateStoreCorruptionError(`snapshot generation ${generation} has a scope digest mismatch`);
|
|
1478
1568
|
}
|
|
1479
1569
|
snapshotReadHookForTest?.();
|
|
1480
|
-
const
|
|
1481
|
-
|
|
1482
|
-
|
|
1483
|
-
}
|
|
1484
|
-
const checksum = Buffer.alloc(32);
|
|
1485
|
-
if (readIntoSync(fd, checksum, SNAPSHOT_FIXED_HEADER_BYTES + payloadLength) !== checksum.length) {
|
|
1486
|
-
throw new StateStoreCorruptionError(`snapshot generation ${generation} has a torn checksum`);
|
|
1487
|
-
}
|
|
1488
|
-
const expected = crypto.createHash("sha256").update(header).update(data).digest();
|
|
1489
|
-
if (!expected.equals(checksum)) {
|
|
1490
|
-
throw new StateStoreCorruptionError(`snapshot generation ${generation} has a bad checksum`);
|
|
1491
|
-
}
|
|
1492
|
-
let decoded;
|
|
1493
|
-
try {
|
|
1494
|
-
decoded = JSON.parse(data.toString("utf8"));
|
|
1495
|
-
}
|
|
1496
|
-
catch (error) {
|
|
1497
|
-
throw new StateStoreCorruptionError(`snapshot generation ${generation} has invalid JSON`, { cause: error });
|
|
1498
|
-
}
|
|
1570
|
+
const decoded = snapshotDecoder === undefined
|
|
1571
|
+
? decodeSnapshotWholePayload(fd, header, payloadLength, generation)
|
|
1572
|
+
: decodeSnapshotIncrementally(fd, header, payloadLength, generation, snapshotDecoder, magic);
|
|
1499
1573
|
if (!isSnapshotPayload(decoded)) {
|
|
1500
1574
|
throw new StateStoreCorruptionError(`snapshot generation ${generation} has an invalid state payload`);
|
|
1501
1575
|
}
|
|
1502
|
-
return {
|
|
1576
|
+
return {
|
|
1577
|
+
payload: decoded,
|
|
1578
|
+
maxReadBytes: snapshotDecoder === undefined ? payloadLength : Math.min(payloadLength, SNAPSHOT_CHUNK_BYTES),
|
|
1579
|
+
};
|
|
1503
1580
|
}
|
|
1504
1581
|
catch (error) {
|
|
1505
1582
|
if (error instanceof StateStoreCorruptionError)
|
|
@@ -1814,7 +1891,7 @@ function encodeSnapshotChunks(generation, scopeDigest, payload) {
|
|
|
1814
1891
|
writeCanonicalJson(payload, sink);
|
|
1815
1892
|
sink.flush();
|
|
1816
1893
|
const header = Buffer.alloc(SNAPSHOT_FIXED_HEADER_BYTES);
|
|
1817
|
-
|
|
1894
|
+
SNAPSHOT_V3_MAGIC.copy(header, 0);
|
|
1818
1895
|
writeU64(header, generation, 8);
|
|
1819
1896
|
header.writeUInt32BE(sink.byteLength, 16);
|
|
1820
1897
|
scopeDigest.copy(header, 20);
|
|
@@ -1834,7 +1911,7 @@ function encodeSnapshotChunks(generation, scopeDigest, payload) {
|
|
|
1834
1911
|
function encodeSnapshotWholeBuffer(generation, scopeDigest, payload) {
|
|
1835
1912
|
const data = Buffer.from(canonicalJson(payload), "utf8");
|
|
1836
1913
|
const header = Buffer.alloc(SNAPSHOT_FIXED_HEADER_BYTES);
|
|
1837
|
-
|
|
1914
|
+
SNAPSHOT_V3_MAGIC.copy(header, 0);
|
|
1838
1915
|
writeU64(header, generation, 8);
|
|
1839
1916
|
header.writeUInt32BE(data.length, 16);
|
|
1840
1917
|
scopeDigest.copy(header, 20);
|