keyframe-store 4.1.1 → 4.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +5 -5
- package/dist/index.d.ts +0 -82
- package/dist/index.js +0 -175
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "keyframe-store",
|
|
3
|
-
"version": "4.
|
|
3
|
+
"version": "4.2.0",
|
|
4
4
|
"description": "Content-addressed version storage — every version is a zstd keyframe or a delta against one, over any backend that can store bytes by key.",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"compression",
|
|
@@ -52,17 +52,17 @@
|
|
|
52
52
|
"typecheck": "tsc"
|
|
53
53
|
},
|
|
54
54
|
"dependencies": {
|
|
55
|
-
"@esposter/shared": "4.
|
|
55
|
+
"@esposter/shared": "4.2.0",
|
|
56
56
|
"neverthrow": "^8.2.0"
|
|
57
57
|
},
|
|
58
58
|
"devDependencies": {
|
|
59
59
|
"@arethetypeswrong/core": "^0.18.5",
|
|
60
|
-
"@esposter/configuration": "4.
|
|
61
|
-
"@esposter/shared-node": "4.
|
|
60
|
+
"@esposter/configuration": "4.2.0",
|
|
61
|
+
"@esposter/shared-node": "4.2.0",
|
|
62
62
|
"@types/node": "^26.6.5",
|
|
63
63
|
"publint": "^0.3.25",
|
|
64
64
|
"tsdown": "^0.23.1",
|
|
65
65
|
"vitest": "^5.0.4"
|
|
66
66
|
},
|
|
67
|
-
"gitHead": "
|
|
67
|
+
"gitHead": "fef4971cf6a621a2c1850f32fcec19ee4ec0de99"
|
|
68
68
|
}
|
package/dist/index.d.ts
DELETED
|
@@ -1,82 +0,0 @@
|
|
|
1
|
-
import { InvalidOperationError } from "@esposter/shared";
|
|
2
|
-
import { ResultAsync } from "neverthrow";
|
|
3
|
-
//#region src/constants.d.ts
|
|
4
|
-
export declare const OBJECT_MAGIC: Uint8Array;
|
|
5
|
-
export declare const OBJECT_FORMAT_VERSION = 1;
|
|
6
|
-
export declare const OBJECT_VERSION_OFFSET = 4;
|
|
7
|
-
export declare const OBJECT_FLAGS_OFFSET = 5;
|
|
8
|
-
export declare const OBJECT_WINDOW_LOG_OFFSET = 6;
|
|
9
|
-
export declare const OBJECT_HEADER_BYTE_COUNT = 8;
|
|
10
|
-
export declare const KEYFRAME_FLAG = 0;
|
|
11
|
-
export declare const DELTA_FLAG = 1;
|
|
12
|
-
export declare const HASH_BYTE_COUNT = 32;
|
|
13
|
-
export declare const DELTA_HEADER_BYTE_COUNT: number;
|
|
14
|
-
export declare const DEFAULT_PROMOTION_RATIO: number;
|
|
15
|
-
export declare const DEFAULT_SEGMENT_BUDGET_RATIO = 1;
|
|
16
|
-
//#endregion
|
|
17
|
-
//#region src/models/VersionAnchor.d.ts
|
|
18
|
-
export interface VersionAnchor {
|
|
19
|
-
anchoredBytes: number;
|
|
20
|
-
hash: string;
|
|
21
|
-
}
|
|
22
|
-
//#endregion
|
|
23
|
-
//#region src/models/WrittenVersion.d.ts
|
|
24
|
-
export interface WrittenVersion {
|
|
25
|
-
baseHash: string;
|
|
26
|
-
hash: string;
|
|
27
|
-
isDeduplicated: boolean;
|
|
28
|
-
plaintextBytes: number;
|
|
29
|
-
storedBytes: number;
|
|
30
|
-
}
|
|
31
|
-
//#endregion
|
|
32
|
-
//#region src/models/KeyframeStore.d.ts
|
|
33
|
-
export interface KeyframeStore {
|
|
34
|
-
collect: (releasedHashes: string[], retainedHashes: string[]) => ResultAsync<string[], Error>;
|
|
35
|
-
read: (hash: string) => ResultAsync<Uint8Array, Error>;
|
|
36
|
-
write: (plaintext: Uint8Array, anchor: VersionAnchor) => ResultAsync<WrittenVersion, Error>;
|
|
37
|
-
}
|
|
38
|
-
//#endregion
|
|
39
|
-
//#region src/models/KeyframeStoreOptions.d.ts
|
|
40
|
-
export interface KeyframeStoreOptions {
|
|
41
|
-
compressionLevel?: number;
|
|
42
|
-
promotionRatio?: number;
|
|
43
|
-
segmentBudgetRatio?: number;
|
|
44
|
-
}
|
|
45
|
-
//#endregion
|
|
46
|
-
//#region src/models/ObjectStore.d.ts
|
|
47
|
-
export interface ObjectStore {
|
|
48
|
-
delete: (keys: string[]) => Promise<void>;
|
|
49
|
-
read: (key: string) => Promise<Uint8Array | undefined>;
|
|
50
|
-
write: (key: string, bytes: Uint8Array) => Promise<boolean>;
|
|
51
|
-
}
|
|
52
|
-
//#endregion
|
|
53
|
-
//#region src/createKeyframeStore.d.ts
|
|
54
|
-
export declare const createKeyframeStore: (objectStore: ObjectStore, { compressionLevel, promotionRatio, segmentBudgetRatio }?: KeyframeStoreOptions) => KeyframeStore;
|
|
55
|
-
//#endregion
|
|
56
|
-
//#region src/models/ObjectNotStoredError.d.ts
|
|
57
|
-
export declare class ObjectNotStoredError extends InvalidOperationError {
|
|
58
|
-
constructor(hash: string, message: string);
|
|
59
|
-
}
|
|
60
|
-
//#endregion
|
|
61
|
-
//#region src/models/ParsedObject.d.ts
|
|
62
|
-
export interface ParsedObject {
|
|
63
|
-
baseHash: string;
|
|
64
|
-
payload: Uint8Array;
|
|
65
|
-
windowLog: number;
|
|
66
|
-
}
|
|
67
|
-
//#endregion
|
|
68
|
-
//#region src/services/decodeObject.d.ts
|
|
69
|
-
export declare const decodeObject: (parsedObject: ParsedObject, basePlaintext?: Uint8Array) => Promise<Buffer>;
|
|
70
|
-
//#endregion
|
|
71
|
-
//#region src/services/encodeObject.d.ts
|
|
72
|
-
export declare const encodeObject: (plaintext: Uint8Array, compressionLevel: number, base?: {
|
|
73
|
-
hash: string;
|
|
74
|
-
plaintext: Uint8Array;
|
|
75
|
-
}) => Promise<Uint8Array>;
|
|
76
|
-
//#endregion
|
|
77
|
-
//#region src/services/getContentAddress.d.ts
|
|
78
|
-
export declare const getContentAddress: (plaintext: Uint8Array) => string;
|
|
79
|
-
//#endregion
|
|
80
|
-
//#region src/services/parseObject.d.ts
|
|
81
|
-
export declare const parseObject: (hash: string, bytes: Uint8Array) => ParsedObject;
|
|
82
|
-
//#endregion
|
package/dist/index.js
DELETED
|
@@ -1,175 +0,0 @@
|
|
|
1
|
-
import { DEFAULT_COMPRESSION_LEVEL, InvalidOperationError, Operation, getResultAsync, getWindowLog } from "@esposter/shared";
|
|
2
|
-
import { promisify } from "node:util";
|
|
3
|
-
import { constants, zstdCompress, zstdDecompress } from "node:zlib";
|
|
4
|
-
import { createHash } from "node:crypto";
|
|
5
|
-
//#region src/constants.ts
|
|
6
|
-
const OBJECT_MAGIC = new Uint8Array([
|
|
7
|
-
75,
|
|
8
|
-
70,
|
|
9
|
-
83,
|
|
10
|
-
84
|
|
11
|
-
]);
|
|
12
|
-
const OBJECT_FORMAT_VERSION = 1;
|
|
13
|
-
const OBJECT_VERSION_OFFSET = 4;
|
|
14
|
-
const OBJECT_FLAGS_OFFSET = 5;
|
|
15
|
-
const OBJECT_WINDOW_LOG_OFFSET = 6;
|
|
16
|
-
const OBJECT_HEADER_BYTE_COUNT = 8;
|
|
17
|
-
const KEYFRAME_FLAG = 0;
|
|
18
|
-
const DELTA_FLAG = 1;
|
|
19
|
-
const HASH_BYTE_COUNT = 32;
|
|
20
|
-
const DELTA_HEADER_BYTE_COUNT = 40;
|
|
21
|
-
const DEFAULT_PROMOTION_RATIO = 1 / 3;
|
|
22
|
-
const DEFAULT_SEGMENT_BUDGET_RATIO = 1;
|
|
23
|
-
//#endregion
|
|
24
|
-
//#region src/models/ObjectNotStoredError.ts
|
|
25
|
-
var ObjectNotStoredError = class extends InvalidOperationError {
|
|
26
|
-
constructor(hash, message) {
|
|
27
|
-
super(Operation.Read, hash, message);
|
|
28
|
-
this.name = "ObjectNotStoredError";
|
|
29
|
-
}
|
|
30
|
-
};
|
|
31
|
-
//#endregion
|
|
32
|
-
//#region src/services/decodeObject.ts
|
|
33
|
-
const decompress = promisify(zstdDecompress);
|
|
34
|
-
const decodeObject = (parsedObject, basePlaintext) => decompress(parsedObject.payload, {
|
|
35
|
-
...basePlaintext ? { dictionary: basePlaintext } : {},
|
|
36
|
-
params: { [constants.ZSTD_d_windowLogMax]: parsedObject.windowLog }
|
|
37
|
-
});
|
|
38
|
-
//#endregion
|
|
39
|
-
//#region src/services/encodeObject.ts
|
|
40
|
-
const compress = promisify(zstdCompress);
|
|
41
|
-
const encodeObject = async (plaintext, compressionLevel, base) => {
|
|
42
|
-
const windowLog = getWindowLog(base?.plaintext.byteLength ?? 0, plaintext.byteLength);
|
|
43
|
-
const payload = await compress(plaintext, {
|
|
44
|
-
...base ? { dictionary: base.plaintext } : {},
|
|
45
|
-
params: {
|
|
46
|
-
[constants.ZSTD_c_compressionLevel]: compressionLevel,
|
|
47
|
-
[constants.ZSTD_c_windowLog]: windowLog
|
|
48
|
-
}
|
|
49
|
-
});
|
|
50
|
-
const headerByteCount = base ? 40 : 8;
|
|
51
|
-
const bytes = new Uint8Array(headerByteCount + payload.byteLength);
|
|
52
|
-
bytes.set(OBJECT_MAGIC);
|
|
53
|
-
bytes[4] = 1;
|
|
54
|
-
bytes[5] = base ? 1 : 0;
|
|
55
|
-
bytes[6] = windowLog;
|
|
56
|
-
if (base) bytes.set(Buffer.from(base.hash, "hex").subarray(0, 32), 8);
|
|
57
|
-
bytes.set(payload, headerByteCount);
|
|
58
|
-
return bytes;
|
|
59
|
-
};
|
|
60
|
-
//#endregion
|
|
61
|
-
//#region src/services/getContentAddress.ts
|
|
62
|
-
const getContentAddress = (plaintext) => createHash("sha256").update(plaintext).digest("hex");
|
|
63
|
-
//#endregion
|
|
64
|
-
//#region src/services/parseObject.ts
|
|
65
|
-
const parseObject = (hash, bytes) => {
|
|
66
|
-
if (!(bytes.byteLength >= 8 && OBJECT_MAGIC.every((byte, index) => bytes[index] === byte))) throw new InvalidOperationError(Operation.Read, hash, "not a keyframe store object");
|
|
67
|
-
else if (bytes[4] !== 1) throw new InvalidOperationError(Operation.Read, hash, `unknown object format version ${bytes[4]}`);
|
|
68
|
-
const windowLog = bytes[6] ?? 0;
|
|
69
|
-
const flags = bytes[5];
|
|
70
|
-
if (flags === 0) return {
|
|
71
|
-
baseHash: "",
|
|
72
|
-
payload: bytes.subarray(8),
|
|
73
|
-
windowLog
|
|
74
|
-
};
|
|
75
|
-
else if (flags !== 1) throw new InvalidOperationError(Operation.Read, hash, `unknown object flags ${flags}`);
|
|
76
|
-
else if (bytes.byteLength < 40) throw new InvalidOperationError(Operation.Read, hash, "delta object is truncated before its base hash");
|
|
77
|
-
return {
|
|
78
|
-
baseHash: Buffer.from(bytes.subarray(8, 40)).toString("hex"),
|
|
79
|
-
payload: bytes.subarray(40),
|
|
80
|
-
windowLog
|
|
81
|
-
};
|
|
82
|
-
};
|
|
83
|
-
//#endregion
|
|
84
|
-
//#region src/createKeyframeStore.ts
|
|
85
|
-
const decodeVerifiedObject = async (hash, parsedObject, basePlaintext) => {
|
|
86
|
-
const plaintext = await decodeObject(parsedObject, basePlaintext);
|
|
87
|
-
if (getContentAddress(plaintext) !== hash) throw new InvalidOperationError(Operation.Read, hash, "object does not hash to its key");
|
|
88
|
-
return plaintext;
|
|
89
|
-
};
|
|
90
|
-
const createKeyframeStore = (objectStore, { compressionLevel = DEFAULT_COMPRESSION_LEVEL, promotionRatio = DEFAULT_PROMOTION_RATIO, segmentBudgetRatio = 1 } = {}) => {
|
|
91
|
-
const readKeyframe = async (hash) => {
|
|
92
|
-
const bytes = await objectStore.read(hash);
|
|
93
|
-
if (!bytes) return void 0;
|
|
94
|
-
const parsedObject = parseObject(hash, bytes);
|
|
95
|
-
if (parsedObject.baseHash) throw new InvalidOperationError(Operation.Read, hash, `is a delta against ${parsedObject.baseHash}, not a keyframe`);
|
|
96
|
-
return {
|
|
97
|
-
plaintext: await decodeVerifiedObject(hash, parsedObject),
|
|
98
|
-
storedBytes: bytes.byteLength
|
|
99
|
-
};
|
|
100
|
-
};
|
|
101
|
-
const readStoredObject = async (hash) => {
|
|
102
|
-
const bytes = await objectStore.read(hash);
|
|
103
|
-
if (!bytes) return void 0;
|
|
104
|
-
const parsedObject = parseObject(hash, bytes);
|
|
105
|
-
if (!parsedObject.baseHash) return {
|
|
106
|
-
baseHash: "",
|
|
107
|
-
plaintext: await decodeVerifiedObject(hash, parsedObject)
|
|
108
|
-
};
|
|
109
|
-
const keyframe = await readKeyframe(parsedObject.baseHash);
|
|
110
|
-
if (!keyframe) throw new ObjectNotStoredError(hash, `keyframe ${parsedObject.baseHash} is not stored`);
|
|
111
|
-
return {
|
|
112
|
-
baseHash: parsedObject.baseHash,
|
|
113
|
-
plaintext: await decodeVerifiedObject(hash, parsedObject, keyframe.plaintext)
|
|
114
|
-
};
|
|
115
|
-
};
|
|
116
|
-
return {
|
|
117
|
-
collect: (releasedHashes, retainedHashes) => getResultAsync(async () => {
|
|
118
|
-
const retainedHashSet = new Set(retainedHashes);
|
|
119
|
-
const collectableHashes = [...new Set(releasedHashes)].filter((hash) => !retainedHashSet.has(hash));
|
|
120
|
-
if (collectableHashes.length > 0) await objectStore.delete(collectableHashes);
|
|
121
|
-
return collectableHashes;
|
|
122
|
-
}),
|
|
123
|
-
read: (hash) => getResultAsync(async () => {
|
|
124
|
-
const storedObject = await readStoredObject(hash);
|
|
125
|
-
if (!storedObject) throw new ObjectNotStoredError(hash, "object is not stored");
|
|
126
|
-
return storedObject.plaintext;
|
|
127
|
-
}),
|
|
128
|
-
write: (plaintext, anchor) => getResultAsync(async () => {
|
|
129
|
-
const hash = getContentAddress(plaintext);
|
|
130
|
-
const plaintextBytes = plaintext.byteLength;
|
|
131
|
-
const adoptStored = (baseHash) => ({
|
|
132
|
-
baseHash,
|
|
133
|
-
hash,
|
|
134
|
-
isDeduplicated: true,
|
|
135
|
-
plaintextBytes,
|
|
136
|
-
storedBytes: 0
|
|
137
|
-
});
|
|
138
|
-
const storedObject = await readStoredObject(hash);
|
|
139
|
-
if (storedObject) return adoptStored(storedObject.baseHash);
|
|
140
|
-
const writeObject = async (bytes, baseHash) => {
|
|
141
|
-
if (await objectStore.write(hash, bytes)) return {
|
|
142
|
-
baseHash,
|
|
143
|
-
hash,
|
|
144
|
-
isDeduplicated: false,
|
|
145
|
-
plaintextBytes,
|
|
146
|
-
storedBytes: bytes.byteLength
|
|
147
|
-
};
|
|
148
|
-
const twinObject = await readStoredObject(hash);
|
|
149
|
-
if (!twinObject) throw new InvalidOperationError(Operation.Create, hash, "refused as already stored, but nothing is stored");
|
|
150
|
-
return adoptStored(twinObject.baseHash);
|
|
151
|
-
};
|
|
152
|
-
const encodeDelta = async () => {
|
|
153
|
-
const keyframe = await readKeyframe(anchor.hash);
|
|
154
|
-
if (!keyframe) return void 0;
|
|
155
|
-
return {
|
|
156
|
-
bytes: await encodeObject(plaintext, compressionLevel, {
|
|
157
|
-
hash: anchor.hash,
|
|
158
|
-
plaintext: keyframe.plaintext
|
|
159
|
-
}),
|
|
160
|
-
keyframeStoredBytes: keyframe.storedBytes
|
|
161
|
-
};
|
|
162
|
-
};
|
|
163
|
-
const keyframeEncoding = encodeObject(plaintext, compressionLevel);
|
|
164
|
-
const deltaEncoding = anchor.hash ? encodeDelta() : void 0;
|
|
165
|
-
const [keyframeBytes, delta] = await Promise.all([keyframeEncoding, deltaEncoding]);
|
|
166
|
-
if (!delta) return writeObject(keyframeBytes, "");
|
|
167
|
-
const isWithinPromotionRatio = delta.bytes.byteLength <= keyframeBytes.byteLength * promotionRatio;
|
|
168
|
-
const isWithinSegmentBudget = anchor.anchoredBytes + delta.bytes.byteLength <= delta.keyframeStoredBytes * segmentBudgetRatio;
|
|
169
|
-
if (!isWithinPromotionRatio || !isWithinSegmentBudget) return writeObject(keyframeBytes, "");
|
|
170
|
-
else return writeObject(delta.bytes, anchor.hash);
|
|
171
|
-
})
|
|
172
|
-
};
|
|
173
|
-
};
|
|
174
|
-
//#endregion
|
|
175
|
-
export { DEFAULT_PROMOTION_RATIO, DEFAULT_SEGMENT_BUDGET_RATIO, DELTA_FLAG, DELTA_HEADER_BYTE_COUNT, HASH_BYTE_COUNT, KEYFRAME_FLAG, OBJECT_FLAGS_OFFSET, OBJECT_FORMAT_VERSION, OBJECT_HEADER_BYTE_COUNT, OBJECT_MAGIC, OBJECT_VERSION_OFFSET, OBJECT_WINDOW_LOG_OFFSET, ObjectNotStoredError, createKeyframeStore, decodeObject, encodeObject, getContentAddress, parseObject };
|