keyframe-store 4.1.2 → 4.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (3) hide show
  1. package/package.json +5 -5
  2. package/dist/index.d.ts +0 -82
  3. package/dist/index.js +0 -175
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "keyframe-store",
3
- "version": "4.1.2",
3
+ "version": "4.2.0",
4
4
  "description": "Content-addressed version storage — every version is a zstd keyframe or a delta against one, over any backend that can store bytes by key.",
5
5
  "keywords": [
6
6
  "compression",
@@ -52,17 +52,17 @@
52
52
  "typecheck": "tsc"
53
53
  },
54
54
  "dependencies": {
55
- "@esposter/shared": "4.1.2",
55
+ "@esposter/shared": "4.2.0",
56
56
  "neverthrow": "^8.2.0"
57
57
  },
58
58
  "devDependencies": {
59
59
  "@arethetypeswrong/core": "^0.18.5",
60
- "@esposter/configuration": "4.1.2",
61
- "@esposter/shared-node": "4.1.2",
60
+ "@esposter/configuration": "4.2.0",
61
+ "@esposter/shared-node": "4.2.0",
62
62
  "@types/node": "^26.6.5",
63
63
  "publint": "^0.3.25",
64
64
  "tsdown": "^0.23.1",
65
65
  "vitest": "^5.0.4"
66
66
  },
67
- "gitHead": "144e3f98570b015e3181bfbc6cb70d346ce826b5"
67
+ "gitHead": "fef4971cf6a621a2c1850f32fcec19ee4ec0de99"
68
68
  }
package/dist/index.d.ts DELETED
@@ -1,82 +0,0 @@
1
- import { InvalidOperationError } from "@esposter/shared";
2
- import { ResultAsync } from "neverthrow";
3
- //#region src/constants.d.ts
4
- export declare const OBJECT_MAGIC: Uint8Array;
5
- export declare const OBJECT_FORMAT_VERSION = 1;
6
- export declare const OBJECT_VERSION_OFFSET = 4;
7
- export declare const OBJECT_FLAGS_OFFSET = 5;
8
- export declare const OBJECT_WINDOW_LOG_OFFSET = 6;
9
- export declare const OBJECT_HEADER_BYTE_COUNT = 8;
10
- export declare const KEYFRAME_FLAG = 0;
11
- export declare const DELTA_FLAG = 1;
12
- export declare const HASH_BYTE_COUNT = 32;
13
- export declare const DELTA_HEADER_BYTE_COUNT: number;
14
- export declare const DEFAULT_PROMOTION_RATIO: number;
15
- export declare const DEFAULT_SEGMENT_BUDGET_RATIO = 1;
16
- //#endregion
17
- //#region src/models/VersionAnchor.d.ts
18
- export interface VersionAnchor {
19
- anchoredBytes: number;
20
- hash: string;
21
- }
22
- //#endregion
23
- //#region src/models/WrittenVersion.d.ts
24
- export interface WrittenVersion {
25
- baseHash: string;
26
- hash: string;
27
- isDeduplicated: boolean;
28
- plaintextBytes: number;
29
- storedBytes: number;
30
- }
31
- //#endregion
32
- //#region src/models/KeyframeStore.d.ts
33
- export interface KeyframeStore {
34
- collect: (releasedHashes: string[], retainedHashes: string[]) => ResultAsync<string[], Error>;
35
- read: (hash: string) => ResultAsync<Uint8Array, Error>;
36
- write: (plaintext: Uint8Array, anchor: VersionAnchor) => ResultAsync<WrittenVersion, Error>;
37
- }
38
- //#endregion
39
- //#region src/models/KeyframeStoreOptions.d.ts
40
- export interface KeyframeStoreOptions {
41
- compressionLevel?: number;
42
- promotionRatio?: number;
43
- segmentBudgetRatio?: number;
44
- }
45
- //#endregion
46
- //#region src/models/ObjectStore.d.ts
47
- export interface ObjectStore {
48
- delete: (keys: string[]) => Promise<void>;
49
- read: (key: string) => Promise<Uint8Array | undefined>;
50
- write: (key: string, bytes: Uint8Array) => Promise<boolean>;
51
- }
52
- //#endregion
53
- //#region src/createKeyframeStore.d.ts
54
- export declare const createKeyframeStore: (objectStore: ObjectStore, { compressionLevel, promotionRatio, segmentBudgetRatio }?: KeyframeStoreOptions) => KeyframeStore;
55
- //#endregion
56
- //#region src/models/ObjectNotStoredError.d.ts
57
- export declare class ObjectNotStoredError extends InvalidOperationError {
58
- constructor(hash: string, message: string);
59
- }
60
- //#endregion
61
- //#region src/models/ParsedObject.d.ts
62
- export interface ParsedObject {
63
- baseHash: string;
64
- payload: Uint8Array;
65
- windowLog: number;
66
- }
67
- //#endregion
68
- //#region src/services/decodeObject.d.ts
69
- export declare const decodeObject: (parsedObject: ParsedObject, basePlaintext?: Uint8Array) => Promise<Buffer>;
70
- //#endregion
71
- //#region src/services/encodeObject.d.ts
72
- export declare const encodeObject: (plaintext: Uint8Array, compressionLevel: number, base?: {
73
- hash: string;
74
- plaintext: Uint8Array;
75
- }) => Promise<Uint8Array>;
76
- //#endregion
77
- //#region src/services/getContentAddress.d.ts
78
- export declare const getContentAddress: (plaintext: Uint8Array) => string;
79
- //#endregion
80
- //#region src/services/parseObject.d.ts
81
- export declare const parseObject: (hash: string, bytes: Uint8Array) => ParsedObject;
82
- //#endregion
package/dist/index.js DELETED
@@ -1,175 +0,0 @@
1
- import { DEFAULT_COMPRESSION_LEVEL, InvalidOperationError, Operation, getResultAsync, getWindowLog } from "@esposter/shared";
2
- import { promisify } from "node:util";
3
- import { constants, zstdCompress, zstdDecompress } from "node:zlib";
4
- import { createHash } from "node:crypto";
5
- //#region src/constants.ts
6
- const OBJECT_MAGIC = new Uint8Array([
7
- 75,
8
- 70,
9
- 83,
10
- 84
11
- ]);
12
- const OBJECT_FORMAT_VERSION = 1;
13
- const OBJECT_VERSION_OFFSET = 4;
14
- const OBJECT_FLAGS_OFFSET = 5;
15
- const OBJECT_WINDOW_LOG_OFFSET = 6;
16
- const OBJECT_HEADER_BYTE_COUNT = 8;
17
- const KEYFRAME_FLAG = 0;
18
- const DELTA_FLAG = 1;
19
- const HASH_BYTE_COUNT = 32;
20
- const DELTA_HEADER_BYTE_COUNT = 40;
21
- const DEFAULT_PROMOTION_RATIO = 1 / 3;
22
- const DEFAULT_SEGMENT_BUDGET_RATIO = 1;
23
- //#endregion
24
- //#region src/models/ObjectNotStoredError.ts
25
- var ObjectNotStoredError = class extends InvalidOperationError {
26
- constructor(hash, message) {
27
- super(Operation.Read, hash, message);
28
- this.name = "ObjectNotStoredError";
29
- }
30
- };
31
- //#endregion
32
- //#region src/services/decodeObject.ts
33
- const decompress = promisify(zstdDecompress);
34
- const decodeObject = (parsedObject, basePlaintext) => decompress(parsedObject.payload, {
35
- ...basePlaintext ? { dictionary: basePlaintext } : {},
36
- params: { [constants.ZSTD_d_windowLogMax]: parsedObject.windowLog }
37
- });
38
- //#endregion
39
- //#region src/services/encodeObject.ts
40
- const compress = promisify(zstdCompress);
41
- const encodeObject = async (plaintext, compressionLevel, base) => {
42
- const windowLog = getWindowLog(base?.plaintext.byteLength ?? 0, plaintext.byteLength);
43
- const payload = await compress(plaintext, {
44
- ...base ? { dictionary: base.plaintext } : {},
45
- params: {
46
- [constants.ZSTD_c_compressionLevel]: compressionLevel,
47
- [constants.ZSTD_c_windowLog]: windowLog
48
- }
49
- });
50
- const headerByteCount = base ? 40 : 8;
51
- const bytes = new Uint8Array(headerByteCount + payload.byteLength);
52
- bytes.set(OBJECT_MAGIC);
53
- bytes[4] = 1;
54
- bytes[5] = base ? 1 : 0;
55
- bytes[6] = windowLog;
56
- if (base) bytes.set(Buffer.from(base.hash, "hex").subarray(0, 32), 8);
57
- bytes.set(payload, headerByteCount);
58
- return bytes;
59
- };
60
- //#endregion
61
- //#region src/services/getContentAddress.ts
62
- const getContentAddress = (plaintext) => createHash("sha256").update(plaintext).digest("hex");
63
- //#endregion
64
- //#region src/services/parseObject.ts
65
- const parseObject = (hash, bytes) => {
66
- if (!(bytes.byteLength >= 8 && OBJECT_MAGIC.every((byte, index) => bytes[index] === byte))) throw new InvalidOperationError(Operation.Read, hash, "not a keyframe store object");
67
- else if (bytes[4] !== 1) throw new InvalidOperationError(Operation.Read, hash, `unknown object format version ${bytes[4]}`);
68
- const windowLog = bytes[6] ?? 0;
69
- const flags = bytes[5];
70
- if (flags === 0) return {
71
- baseHash: "",
72
- payload: bytes.subarray(8),
73
- windowLog
74
- };
75
- else if (flags !== 1) throw new InvalidOperationError(Operation.Read, hash, `unknown object flags ${flags}`);
76
- else if (bytes.byteLength < 40) throw new InvalidOperationError(Operation.Read, hash, "delta object is truncated before its base hash");
77
- return {
78
- baseHash: Buffer.from(bytes.subarray(8, 40)).toString("hex"),
79
- payload: bytes.subarray(40),
80
- windowLog
81
- };
82
- };
83
- //#endregion
84
- //#region src/createKeyframeStore.ts
85
- const decodeVerifiedObject = async (hash, parsedObject, basePlaintext) => {
86
- const plaintext = await decodeObject(parsedObject, basePlaintext);
87
- if (getContentAddress(plaintext) !== hash) throw new InvalidOperationError(Operation.Read, hash, "object does not hash to its key");
88
- return plaintext;
89
- };
90
- const createKeyframeStore = (objectStore, { compressionLevel = DEFAULT_COMPRESSION_LEVEL, promotionRatio = DEFAULT_PROMOTION_RATIO, segmentBudgetRatio = 1 } = {}) => {
91
- const readKeyframe = async (hash) => {
92
- const bytes = await objectStore.read(hash);
93
- if (!bytes) return void 0;
94
- const parsedObject = parseObject(hash, bytes);
95
- if (parsedObject.baseHash) throw new InvalidOperationError(Operation.Read, hash, `is a delta against ${parsedObject.baseHash}, not a keyframe`);
96
- return {
97
- plaintext: await decodeVerifiedObject(hash, parsedObject),
98
- storedBytes: bytes.byteLength
99
- };
100
- };
101
- const readStoredObject = async (hash) => {
102
- const bytes = await objectStore.read(hash);
103
- if (!bytes) return void 0;
104
- const parsedObject = parseObject(hash, bytes);
105
- if (!parsedObject.baseHash) return {
106
- baseHash: "",
107
- plaintext: await decodeVerifiedObject(hash, parsedObject)
108
- };
109
- const keyframe = await readKeyframe(parsedObject.baseHash);
110
- if (!keyframe) throw new ObjectNotStoredError(hash, `keyframe ${parsedObject.baseHash} is not stored`);
111
- return {
112
- baseHash: parsedObject.baseHash,
113
- plaintext: await decodeVerifiedObject(hash, parsedObject, keyframe.plaintext)
114
- };
115
- };
116
- return {
117
- collect: (releasedHashes, retainedHashes) => getResultAsync(async () => {
118
- const retainedHashSet = new Set(retainedHashes);
119
- const collectableHashes = [...new Set(releasedHashes)].filter((hash) => !retainedHashSet.has(hash));
120
- if (collectableHashes.length > 0) await objectStore.delete(collectableHashes);
121
- return collectableHashes;
122
- }),
123
- read: (hash) => getResultAsync(async () => {
124
- const storedObject = await readStoredObject(hash);
125
- if (!storedObject) throw new ObjectNotStoredError(hash, "object is not stored");
126
- return storedObject.plaintext;
127
- }),
128
- write: (plaintext, anchor) => getResultAsync(async () => {
129
- const hash = getContentAddress(plaintext);
130
- const plaintextBytes = plaintext.byteLength;
131
- const adoptStored = (baseHash) => ({
132
- baseHash,
133
- hash,
134
- isDeduplicated: true,
135
- plaintextBytes,
136
- storedBytes: 0
137
- });
138
- const storedObject = await readStoredObject(hash);
139
- if (storedObject) return adoptStored(storedObject.baseHash);
140
- const writeObject = async (bytes, baseHash) => {
141
- if (await objectStore.write(hash, bytes)) return {
142
- baseHash,
143
- hash,
144
- isDeduplicated: false,
145
- plaintextBytes,
146
- storedBytes: bytes.byteLength
147
- };
148
- const twinObject = await readStoredObject(hash);
149
- if (!twinObject) throw new InvalidOperationError(Operation.Create, hash, "refused as already stored, but nothing is stored");
150
- return adoptStored(twinObject.baseHash);
151
- };
152
- const encodeDelta = async () => {
153
- const keyframe = await readKeyframe(anchor.hash);
154
- if (!keyframe) return void 0;
155
- return {
156
- bytes: await encodeObject(plaintext, compressionLevel, {
157
- hash: anchor.hash,
158
- plaintext: keyframe.plaintext
159
- }),
160
- keyframeStoredBytes: keyframe.storedBytes
161
- };
162
- };
163
- const keyframeEncoding = encodeObject(plaintext, compressionLevel);
164
- const deltaEncoding = anchor.hash ? encodeDelta() : void 0;
165
- const [keyframeBytes, delta] = await Promise.all([keyframeEncoding, deltaEncoding]);
166
- if (!delta) return writeObject(keyframeBytes, "");
167
- const isWithinPromotionRatio = delta.bytes.byteLength <= keyframeBytes.byteLength * promotionRatio;
168
- const isWithinSegmentBudget = anchor.anchoredBytes + delta.bytes.byteLength <= delta.keyframeStoredBytes * segmentBudgetRatio;
169
- if (!isWithinPromotionRatio || !isWithinSegmentBudget) return writeObject(keyframeBytes, "");
170
- else return writeObject(delta.bytes, anchor.hash);
171
- })
172
- };
173
- };
174
- //#endregion
175
- export { DEFAULT_PROMOTION_RATIO, DEFAULT_SEGMENT_BUDGET_RATIO, DELTA_FLAG, DELTA_HEADER_BYTE_COUNT, HASH_BYTE_COUNT, KEYFRAME_FLAG, OBJECT_FLAGS_OFFSET, OBJECT_FORMAT_VERSION, OBJECT_HEADER_BYTE_COUNT, OBJECT_MAGIC, OBJECT_VERSION_OFFSET, OBJECT_WINDOW_LOG_OFFSET, ObjectNotStoredError, createKeyframeStore, decodeObject, encodeObject, getContentAddress, parseObject };