@tanstack/db-ivm 0.1.12 → 0.1.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cjs/hashing/hash.cjs +22 -3
- package/dist/cjs/hashing/hash.cjs.map +1 -1
- package/dist/cjs/hashing/murmur.cjs +15 -15
- package/dist/cjs/hashing/murmur.cjs.map +1 -1
- package/dist/cjs/hashing/murmur.d.cts +1 -1
- package/dist/esm/hashing/hash.js +22 -3
- package/dist/esm/hashing/hash.js.map +1 -1
- package/dist/esm/hashing/murmur.d.ts +1 -1
- package/dist/esm/hashing/murmur.js +15 -15
- package/dist/esm/hashing/murmur.js.map +1 -1
- package/package.json +1 -1
- package/src/hashing/hash.ts +36 -11
- package/src/hashing/murmur.ts +15 -15
|
@@ -12,6 +12,8 @@ const OBJECT_MARKER = murmur.randomHash();
|
|
|
12
12
|
const ARRAY_MARKER = murmur.randomHash();
|
|
13
13
|
const MAP_MARKER = murmur.randomHash();
|
|
14
14
|
const SET_MARKER = murmur.randomHash();
|
|
15
|
+
const UINT8ARRAY_MARKER = murmur.randomHash();
|
|
16
|
+
const UINT8ARRAY_CONTENT_HASH_THRESHOLD = 128;
|
|
15
17
|
const hashCache = /* @__PURE__ */ new WeakMap();
|
|
16
18
|
function hash(input) {
|
|
17
19
|
const hasher = new murmur.MurmurHashStream();
|
|
@@ -26,6 +28,17 @@ function hashObject(input) {
|
|
|
26
28
|
let valueHash;
|
|
27
29
|
if (input instanceof Date) {
|
|
28
30
|
valueHash = hashDate(input);
|
|
31
|
+
} else if (
|
|
32
|
+
// Check if input is a Uint8Array or Buffer
|
|
33
|
+
typeof Buffer !== `undefined` && input instanceof Buffer || input instanceof Uint8Array
|
|
34
|
+
) {
|
|
35
|
+
if (input.byteLength <= UINT8ARRAY_CONTENT_HASH_THRESHOLD) {
|
|
36
|
+
valueHash = hashUint8Array(input);
|
|
37
|
+
} else {
|
|
38
|
+
return cachedReferenceHash(input);
|
|
39
|
+
}
|
|
40
|
+
} else if (input instanceof File) {
|
|
41
|
+
return cachedReferenceHash(input);
|
|
29
42
|
} else {
|
|
30
43
|
let plainObjectInput = input;
|
|
31
44
|
let marker = OBJECT_MARKER;
|
|
@@ -40,9 +53,6 @@ function hashObject(input) {
|
|
|
40
53
|
marker = SET_MARKER;
|
|
41
54
|
plainObjectInput = [...input.entries()];
|
|
42
55
|
}
|
|
43
|
-
if (typeof Buffer !== `undefined` && input instanceof Buffer || input instanceof Uint8Array || input instanceof File) {
|
|
44
|
-
return cachedReferenceHash(input);
|
|
45
|
-
}
|
|
46
56
|
valueHash = hashPlainObject(plainObjectInput, marker);
|
|
47
57
|
}
|
|
48
58
|
hashCache.set(input, valueHash);
|
|
@@ -54,6 +64,15 @@ function hashDate(input) {
|
|
|
54
64
|
hasher.update(input.getTime());
|
|
55
65
|
return hasher.digest();
|
|
56
66
|
}
|
|
67
|
+
function hashUint8Array(input) {
|
|
68
|
+
const hasher = new murmur.MurmurHashStream();
|
|
69
|
+
hasher.update(UINT8ARRAY_MARKER);
|
|
70
|
+
hasher.update(input.byteLength);
|
|
71
|
+
for (let i = 0; i < input.byteLength; i++) {
|
|
72
|
+
hasher.writeByte(input[i]);
|
|
73
|
+
}
|
|
74
|
+
return hasher.digest();
|
|
75
|
+
}
|
|
57
76
|
function hashPlainObject(input, marker) {
|
|
58
77
|
const hasher = new murmur.MurmurHashStream();
|
|
59
78
|
hasher.update(marker);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"hash.cjs","sources":["../../../src/hashing/hash.ts"],"sourcesContent":["import { MurmurHashStream, randomHash } from \"./murmur.js\"\nimport type { Hasher } from \"./murmur.js\"\n\n/*\n * Implementation of structural hashing based on the Composites polyfill implementation:\n * https://github.com/tc39/proposal-composites\n */\n\nconst TRUE = randomHash()\nconst FALSE = randomHash()\nconst NULL = randomHash()\nconst UNDEFINED = randomHash()\nconst KEY = randomHash()\nconst FUNCTIONS = randomHash()\nconst DATE_MARKER = randomHash()\nconst OBJECT_MARKER = randomHash()\nconst ARRAY_MARKER = randomHash()\nconst MAP_MARKER = randomHash()\nconst SET_MARKER = randomHash()\n\nconst hashCache = new WeakMap<object, number>()\n\nexport function hash(input: any): number {\n const hasher = new MurmurHashStream()\n updateHasher(hasher, input)\n return hasher.digest()\n}\n\nfunction hashObject(input: object): number {\n const cachedHash = hashCache.get(input)\n if (cachedHash !== undefined) {\n return cachedHash\n }\n\n let valueHash: number | undefined\n if (input instanceof Date) {\n valueHash = hashDate(input)\n } else
|
|
1
|
+
{"version":3,"file":"hash.cjs","sources":["../../../src/hashing/hash.ts"],"sourcesContent":["import { MurmurHashStream, randomHash } from \"./murmur.js\"\nimport type { Hasher } from \"./murmur.js\"\n\n/*\n * Implementation of structural hashing based on the Composites polyfill implementation:\n * https://github.com/tc39/proposal-composites\n */\n\nconst TRUE = randomHash()\nconst FALSE = randomHash()\nconst NULL = randomHash()\nconst UNDEFINED = randomHash()\nconst KEY = randomHash()\nconst FUNCTIONS = randomHash()\nconst DATE_MARKER = randomHash()\nconst OBJECT_MARKER = randomHash()\nconst ARRAY_MARKER = randomHash()\nconst MAP_MARKER = randomHash()\nconst SET_MARKER = randomHash()\nconst UINT8ARRAY_MARKER = randomHash()\n\n// Maximum byte length for Uint8Arrays to hash by content instead of reference\n// Arrays smaller than this will be hashed by content, allowing proper equality comparisons\n// for small arrays like ULIDs (16 bytes) while still avoiding performance costs for large arrays\nconst UINT8ARRAY_CONTENT_HASH_THRESHOLD = 128\n\nconst hashCache = new WeakMap<object, number>()\n\nexport function hash(input: any): number {\n const hasher = new MurmurHashStream()\n updateHasher(hasher, input)\n return hasher.digest()\n}\n\nfunction hashObject(input: object): number {\n const cachedHash = hashCache.get(input)\n if (cachedHash !== undefined) {\n return cachedHash\n }\n\n let valueHash: number | undefined\n if (input instanceof Date) {\n valueHash = hashDate(input)\n } else if (\n // Check if input is a Uint8Array or Buffer\n (typeof Buffer !== `undefined` && input instanceof Buffer) ||\n input instanceof Uint8Array\n ) {\n // For small Uint8Arrays/Buffers (e.g., ULIDs, UUIDs), hash by content\n // to enable proper equality comparisons. For large arrays, hash by reference\n // to avoid performance costs.\n if (input.byteLength <= UINT8ARRAY_CONTENT_HASH_THRESHOLD) {\n valueHash = hashUint8Array(input)\n } else {\n // Deeply hashing large arrays would be too costly\n // so we track them by reference and cache them in a weak map\n return cachedReferenceHash(input)\n }\n } else if (input instanceof File) {\n // Files are always hashed by reference due to their potentially large size\n return cachedReferenceHash(input)\n } else {\n let plainObjectInput = input\n let marker = OBJECT_MARKER\n\n if (input instanceof Array) {\n marker = ARRAY_MARKER\n }\n\n if (input instanceof Map) {\n marker = MAP_MARKER\n plainObjectInput = [...input.entries()]\n }\n\n if (input instanceof Set) {\n marker = SET_MARKER\n plainObjectInput = [...input.entries()]\n }\n\n valueHash = hashPlainObject(plainObjectInput, marker)\n }\n\n hashCache.set(input, valueHash)\n return valueHash\n}\n\nfunction hashDate(input: Date): number {\n const hasher = new MurmurHashStream()\n hasher.update(DATE_MARKER)\n hasher.update(input.getTime())\n return hasher.digest()\n}\n\nfunction hashUint8Array(input: Uint8Array): number {\n const hasher = new MurmurHashStream()\n hasher.update(UINT8ARRAY_MARKER)\n // Hash the byte length first to differentiate arrays of different sizes\n hasher.update(input.byteLength)\n // Hash each byte in the array\n for (let i = 0; i < input.byteLength; i++) {\n hasher.writeByte(input[i]!)\n }\n return hasher.digest()\n}\n\nfunction hashPlainObject(input: object, marker: number): number {\n const hasher = new MurmurHashStream()\n\n // Mark the type of the input\n hasher.update(marker)\n const keys = Object.keys(input)\n keys.sort(keySort)\n for (const key of keys) {\n hasher.update(KEY)\n hasher.update(key)\n updateHasher(hasher, input[key as keyof typeof input])\n }\n\n return hasher.digest()\n}\n\nfunction updateHasher(hasher: Hasher, input: unknown): void {\n if (input === null) {\n hasher.update(NULL)\n return\n }\n switch (typeof input) {\n case `undefined`:\n hasher.update(UNDEFINED)\n return\n case `boolean`:\n hasher.update(input ? TRUE : FALSE)\n return\n case `number`:\n // Normalize NaNs and -0\n hasher.update(isNaN(input) ? NaN : input === 0 ? 0 : input)\n return\n case `bigint`:\n case `string`:\n case `symbol`:\n hasher.update(input)\n return\n case `object`:\n hasher.update(getCachedHash(input))\n return\n case `function`:\n // Functions are assigned a globally unique ID\n // and that ID is cached in the weak map\n hasher.update(cachedReferenceHash(input))\n return\n default:\n console.warn(\n `Ignored input during hashing because it is of type ${typeof input} which is not supported`\n )\n }\n}\n\nfunction getCachedHash(input: object): number {\n let valueHash = hashCache.get(input)\n if (valueHash === undefined) {\n valueHash = hashObject(input)\n }\n return valueHash\n}\n\nlet nextRefId = 1\nfunction cachedReferenceHash(fn: object): number {\n let valueHash = hashCache.get(fn)\n if (valueHash === undefined) {\n valueHash = nextRefId ^ FUNCTIONS\n nextRefId++\n hashCache.set(fn, valueHash)\n }\n return valueHash\n}\n\n/**\n * Strings sorted lexicographically.\n */\nfunction keySort(a: string, b: string): number {\n return a.localeCompare(b)\n}\n"],"names":["randomHash","MurmurHashStream"],"mappings":";;;AAQA,MAAM,OAAOA,OAAAA,WAAA;AACb,MAAM,QAAQA,OAAAA,WAAA;AACd,MAAM,OAAOA,OAAAA,WAAA;AACb,MAAM,YAAYA,OAAAA,WAAA;AAClB,MAAM,MAAMA,OAAAA,WAAA;AACZ,MAAM,YAAYA,OAAAA,WAAA;AAClB,MAAM,cAAcA,OAAAA,WAAA;AACpB,MAAM,gBAAgBA,OAAAA,WAAA;AACtB,MAAM,eAAeA,OAAAA,WAAA;AACrB,MAAM,aAAaA,OAAAA,WAAA;AACnB,MAAM,aAAaA,OAAAA,WAAA;AACnB,MAAM,oBAAoBA,OAAAA,WAAA;AAK1B,MAAM,oCAAoC;AAE1C,MAAM,gCAAgB,QAAA;AAEf,SAAS,KAAK,OAAoB;AACvC,QAAM,SAAS,IAAIC,wBAAA;AACnB,eAAa,QAAQ,KAAK;AAC1B,SAAO,OAAO,OAAA;AAChB;AAEA,SAAS,WAAW,OAAuB;AACzC,QAAM,aAAa,UAAU,IAAI,KAAK;AACtC,MAAI,eAAe,QAAW;AAC5B,WAAO;AAAA,EACT;AAEA,MAAI;AACJ,MAAI,iBAAiB,MAAM;AACzB,gBAAY,SAAS,KAAK;AAAA,EAC5B;AAAA;AAAA,IAEG,OAAO,WAAW,eAAe,iBAAiB,UACnD,iBAAiB;AAAA,IACjB;AAIA,QAAI,MAAM,cAAc,mCAAmC;AACzD,kBAAY,eAAe,KAAK;AAAA,IAClC,OAAO;AAGL,aAAO,oBAAoB,KAAK;AAAA,IAClC;AAAA,EACF,WAAW,iBAAiB,MAAM;AAEhC,WAAO,oBAAoB,KAAK;AAAA,EAClC,OAAO;AACL,QAAI,mBAAmB;AACvB,QAAI,SAAS;AAEb,QAAI,iBAAiB,OAAO;AAC1B,eAAS;AAAA,IACX;AAEA,QAAI,iBAAiB,KAAK;AACxB,eAAS;AACT,yBAAmB,CAAC,GAAG,MAAM,SAAS;AAAA,IACxC;AAEA,QAAI,iBAAiB,KAAK;AACxB,eAAS;AACT,yBAAmB,CAAC,GAAG,MAAM,SAAS;AAAA,IACxC;AAEA,gBAAY,gBAAgB,kBAAkB,MAAM;AAAA,EACtD;AAEA,YAAU,IAAI,OAAO,SAAS;AAC9B,SAAO;AACT;AAEA,SAAS,SAAS,OAAqB;AACrC,QAAM,SAAS,IAAIA,wBAAA;AACnB,SAAO,OAAO,WAAW;AACzB,SAAO,OAAO,MAAM,SAAS;AAC7B,SAAO,OAAO,OAAA;AAChB;AAEA,SAAS,eAAe,OAA2B;AACjD,QAAM,SAAS,IAAIA,wBAAA;AACnB,SAAO,OAAO,iBAAiB;AAE/B,SAAO,OAAO,MAAM,UAAU;AAE9B,WAAS,IAAI,GAAG,IAAI,MAAM,YAAY,KAAK;AACzC,WAAO,UAAU,MAAM,CAAC,CAAE;AAAA,EAC5B;AACA,SAAO,OAAO,OAAA;AAChB;AAEA,SAAS,gBAAgB,OAAe,QAAwB;AAC9D,QAAM,SAAS,IAAIA,wBAAA;AAGnB,SAAO,OAAO,MAAM;AACpB,QAAM,OAAO,OAAO,KAAK,KAAK;AAC9B,OAAK,KAAK,OAAO;AACjB,aAAW,OAAO,MAAM;AACtB,WAAO,OAAO,GAAG;AACjB,WAAO,OAAO,GAAG;AACjB,iBAAa,QAAQ,MAAM,GAAyB,CAAC;AAAA,EACvD;AAEA,SAAO,OAAO,OAAA;AAChB;AAEA,SAAS,aAAa,QAAgB,OAAsB;AAC1D,MAAI,UAAU,MAAM;AAClB,WAAO,OAAO,IAAI;AAClB;AAAA,EACF;AACA,UAAQ,OAAO,OAAA;AAAA,IACb,KAAK;AACH,aAAO,OAAO,SAAS;AACvB;AAAA,IACF,KAAK;AACH,aAAO,OAAO,QAAQ,OAAO,KAAK;AAClC;AAAA,IACF,KAAK;AAEH,aAAO,OAAO,MAAM,KAAK,IAAI,MAAM,UAAU,IAAI,IAAI,KAAK;AAC1D;AAAA,IACF,KAAK;AAAA,IACL,KAAK;AAAA,IACL,KAAK;AACH,aAAO,OAAO,KAAK;AACnB;AAAA,IACF,KAAK;AACH,aAAO,OAAO,cAAc,KAAK,CAAC;AAClC;AAAA,IACF,KAAK;AAGH,aAAO,OAAO,oBAAoB,KAAK,CAAC;AACxC;AAAA,IACF;AACE,cAAQ;AAAA,QACN,sDAAsD,OAAO,KAAK;AAAA,MAAA;AAAA,EACpE;AAEN;AAEA,SAAS,cAAc,OAAuB;AAC5C,MAAI,YAAY,UAAU,IAAI,KAAK;AACnC,MAAI,cAAc,QAAW;AAC3B,gBAAY,WAAW,KAAK;AAAA,EAC9B;AACA,SAAO;AACT;AAEA,IAAI,YAAY;AAChB,SAAS,oBAAoB,IAAoB;AAC/C,MAAI,YAAY,UAAU,IAAI,EAAE;AAChC,MAAI,cAAc,QAAW;AAC3B,gBAAY,YAAY;AACxB;AACA,cAAU,IAAI,IAAI,SAAS;AAAA,EAC7B;AACA,SAAO;AACT;AAKA,SAAS,QAAQ,GAAW,GAAmB;AAC7C,SAAO,EAAE,cAAc,CAAC;AAC1B;;"}
|
|
@@ -26,7 +26,7 @@ class MurmurHashStream {
|
|
|
26
26
|
this.hash = this.hash << 13 | this.hash >>> 19;
|
|
27
27
|
this.hash = Math.imul(this.hash, 5) + 3864292196;
|
|
28
28
|
}
|
|
29
|
-
|
|
29
|
+
writeByte(byte) {
|
|
30
30
|
this.carry |= (byte & 255) << 8 * this.carryBytes;
|
|
31
31
|
this.carryBytes++;
|
|
32
32
|
this.length++;
|
|
@@ -46,8 +46,8 @@ class MurmurHashStream {
|
|
|
46
46
|
}
|
|
47
47
|
for (let i = 0; i < description.length; i++) {
|
|
48
48
|
const code = description.charCodeAt(i);
|
|
49
|
-
this.
|
|
50
|
-
this.
|
|
49
|
+
this.writeByte(code & 255);
|
|
50
|
+
this.writeByte(code >>> 8 & 255);
|
|
51
51
|
}
|
|
52
52
|
return;
|
|
53
53
|
}
|
|
@@ -55,20 +55,20 @@ class MurmurHashStream {
|
|
|
55
55
|
this.update(STRING_MARKER);
|
|
56
56
|
for (let i = 0; i < chunk.length; i++) {
|
|
57
57
|
const code = chunk.charCodeAt(i);
|
|
58
|
-
this.
|
|
59
|
-
this.
|
|
58
|
+
this.writeByte(code & 255);
|
|
59
|
+
this.writeByte(code >>> 8 & 255);
|
|
60
60
|
}
|
|
61
61
|
return;
|
|
62
62
|
case `number`:
|
|
63
63
|
dv.setFloat64(0, chunk, true);
|
|
64
|
-
this.
|
|
65
|
-
this.
|
|
66
|
-
this.
|
|
67
|
-
this.
|
|
68
|
-
this.
|
|
69
|
-
this.
|
|
70
|
-
this.
|
|
71
|
-
this.
|
|
64
|
+
this.writeByte(u8[0]);
|
|
65
|
+
this.writeByte(u8[1]);
|
|
66
|
+
this.writeByte(u8[2]);
|
|
67
|
+
this.writeByte(u8[3]);
|
|
68
|
+
this.writeByte(u8[4]);
|
|
69
|
+
this.writeByte(u8[5]);
|
|
70
|
+
this.writeByte(u8[6]);
|
|
71
|
+
this.writeByte(u8[7]);
|
|
72
72
|
return;
|
|
73
73
|
case `bigint`: {
|
|
74
74
|
let value = chunk;
|
|
@@ -79,10 +79,10 @@ class MurmurHashStream {
|
|
|
79
79
|
this.update(BIG_INT_MARKER);
|
|
80
80
|
}
|
|
81
81
|
while (value > 0n) {
|
|
82
|
-
this.
|
|
82
|
+
this.writeByte(Number(value & 0xffn));
|
|
83
83
|
value >>= 8n;
|
|
84
84
|
}
|
|
85
|
-
if (chunk === 0n) this.
|
|
85
|
+
if (chunk === 0n) this.writeByte(0);
|
|
86
86
|
return;
|
|
87
87
|
}
|
|
88
88
|
default:
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"murmur.cjs","sources":["../../../src/hashing/murmur.ts"],"sourcesContent":["/*\n * Implementation of murmur hash based on the Composites polyfill implementation:\n * https://github.com/tc39/proposal-composites\n */\n\nconst RANDOM_SEED = randomHash()\nconst STRING_MARKER = randomHash()\nconst BIG_INT_MARKER = randomHash()\nconst NEG_BIG_INT_MARKER = randomHash()\nconst SYMBOL_MARKER = randomHash()\n\nexport type Hash = number\n\nexport function randomHash() {\n return (Math.random() * (2 ** 31 - 1)) >>> 0\n}\n\nexport interface Hasher {\n update: (val: symbol | string | number | bigint) => void\n digest: () => number\n}\n\n// Allocate these once, outside the hot path\nconst buf = new ArrayBuffer(8)\n// dv and u8 are 2 different views on the same buffer `buf`\nconst dv = new DataView(buf)\nconst u8 = new Uint8Array(buf)\n\n/**\n * This implementation of Murmur hash uses a random initial seed and random markers.\n * This means that hashes aren't deterministic across app restarts.\n * This is intentional in the composites polyfill to be resistent to hash-flooding attacks\n * where malicious users would precompute lots of different objects whose hashes collide with each other.\n *\n * Currently, for ts/db-ivm this is fine because we don't persist client state.\n * However, when we will introduce persistence we will either need to store the seeds or remove the randomness\n * to ensure deterministic hashes across app restarts.\n */\nexport class MurmurHashStream implements Hasher {\n private hash: number = RANDOM_SEED\n private length = 0\n private carry = 0\n private carryBytes = 0\n\n private _mix(k1: number): void {\n k1 = Math.imul(k1, 0xcc9e2d51)\n k1 = (k1 << 15) | (k1 >>> 17)\n k1 = Math.imul(k1, 0x1b873593)\n this.hash ^= k1\n this.hash = (this.hash << 13) | (this.hash >>> 19)\n this.hash = Math.imul(this.hash, 5) + 0xe6546b64\n }\n\n
|
|
1
|
+
{"version":3,"file":"murmur.cjs","sources":["../../../src/hashing/murmur.ts"],"sourcesContent":["/*\n * Implementation of murmur hash based on the Composites polyfill implementation:\n * https://github.com/tc39/proposal-composites\n */\n\nconst RANDOM_SEED = randomHash()\nconst STRING_MARKER = randomHash()\nconst BIG_INT_MARKER = randomHash()\nconst NEG_BIG_INT_MARKER = randomHash()\nconst SYMBOL_MARKER = randomHash()\n\nexport type Hash = number\n\nexport function randomHash() {\n return (Math.random() * (2 ** 31 - 1)) >>> 0\n}\n\nexport interface Hasher {\n update: (val: symbol | string | number | bigint) => void\n digest: () => number\n}\n\n// Allocate these once, outside the hot path\nconst buf = new ArrayBuffer(8)\n// dv and u8 are 2 different views on the same buffer `buf`\nconst dv = new DataView(buf)\nconst u8 = new Uint8Array(buf)\n\n/**\n * This implementation of Murmur hash uses a random initial seed and random markers.\n * This means that hashes aren't deterministic across app restarts.\n * This is intentional in the composites polyfill to be resistent to hash-flooding attacks\n * where malicious users would precompute lots of different objects whose hashes collide with each other.\n *\n * Currently, for ts/db-ivm this is fine because we don't persist client state.\n * However, when we will introduce persistence we will either need to store the seeds or remove the randomness\n * to ensure deterministic hashes across app restarts.\n */\nexport class MurmurHashStream implements Hasher {\n private hash: number = RANDOM_SEED\n private length = 0\n private carry = 0\n private carryBytes = 0\n\n private _mix(k1: number): void {\n k1 = Math.imul(k1, 0xcc9e2d51)\n k1 = (k1 << 15) | (k1 >>> 17)\n k1 = Math.imul(k1, 0x1b873593)\n this.hash ^= k1\n this.hash = (this.hash << 13) | (this.hash >>> 19)\n this.hash = Math.imul(this.hash, 5) + 0xe6546b64\n }\n\n writeByte(byte: number): void {\n this.carry |= (byte & 0xff) << (8 * this.carryBytes)\n this.carryBytes++\n this.length++\n\n if (this.carryBytes === 4) {\n this._mix(this.carry >>> 0)\n this.carry = 0\n this.carryBytes = 0\n }\n }\n\n update(chunk: symbol | string | number | bigint): void {\n switch (typeof chunk) {\n case `symbol`: {\n this.update(SYMBOL_MARKER)\n const description = chunk.description\n if (!description) {\n return\n }\n\n for (let i = 0; i < description.length; i++) {\n const code = description.charCodeAt(i)\n this.writeByte(code & 0xff)\n this.writeByte((code >>> 8) & 0xff)\n }\n return\n }\n case `string`:\n this.update(STRING_MARKER)\n for (let i = 0; i < chunk.length; i++) {\n const code = chunk.charCodeAt(i)\n this.writeByte(code & 0xff)\n this.writeByte((code >>> 8) & 0xff)\n }\n return\n case `number`:\n dv.setFloat64(0, chunk, true) // fixed little-endian\n this.writeByte(u8[0]!)\n this.writeByte(u8[1]!)\n this.writeByte(u8[2]!)\n this.writeByte(u8[3]!)\n this.writeByte(u8[4]!)\n this.writeByte(u8[5]!)\n this.writeByte(u8[6]!)\n this.writeByte(u8[7]!)\n return\n case `bigint`: {\n let value = chunk\n if (value < 0n) {\n value = -value\n this.update(NEG_BIG_INT_MARKER)\n } else {\n this.update(BIG_INT_MARKER)\n }\n while (value > 0n) {\n this.writeByte(Number(value & 0xffn))\n value >>= 8n\n }\n if (chunk === 0n) this.writeByte(0)\n return\n }\n default:\n throw new TypeError(`Unsupported input type: ${typeof chunk}`)\n }\n }\n\n digest(): number {\n if (this.carryBytes > 0) {\n let k1 = this.carry >>> 0\n k1 = Math.imul(k1, 0xcc9e2d51)\n k1 = (k1 << 15) | (k1 >>> 17)\n k1 = Math.imul(k1, 0x1b873593)\n this.hash ^= k1\n }\n\n this.hash ^= this.length\n this.hash ^= this.hash >>> 16\n this.hash = Math.imul(this.hash, 0x85ebca6b)\n this.hash ^= this.hash >>> 13\n this.hash = Math.imul(this.hash, 0xc2b2ae35)\n this.hash ^= this.hash >>> 16\n\n return this.hash >>> 0\n }\n}\n"],"names":[],"mappings":";;AAKA,MAAM,cAAc,WAAA;AACpB,MAAM,gBAAgB,WAAA;AACtB,MAAM,iBAAiB,WAAA;AACvB,MAAM,qBAAqB,WAAA;AAC3B,MAAM,gBAAgB,WAAA;AAIf,SAAS,aAAa;AAC3B,SAAQ,KAAK,OAAA,KAAY,KAAK,KAAK,OAAQ;AAC7C;AAQA,MAAM,MAAM,IAAI,YAAY,CAAC;AAE7B,MAAM,KAAK,IAAI,SAAS,GAAG;AAC3B,MAAM,KAAK,IAAI,WAAW,GAAG;AAYtB,MAAM,iBAAmC;AAAA,EAAzC,cAAA;AACL,SAAQ,OAAe;AACvB,SAAQ,SAAS;AACjB,SAAQ,QAAQ;AAChB,SAAQ,aAAa;AAAA,EAAA;AAAA,EAEb,KAAK,IAAkB;AAC7B,SAAK,KAAK,KAAK,IAAI,UAAU;AAC7B,SAAM,MAAM,KAAO,OAAO;AAC1B,SAAK,KAAK,KAAK,IAAI,SAAU;AAC7B,SAAK,QAAQ;AACb,SAAK,OAAQ,KAAK,QAAQ,KAAO,KAAK,SAAS;AAC/C,SAAK,OAAO,KAAK,KAAK,KAAK,MAAM,CAAC,IAAI;AAAA,EACxC;AAAA,EAEA,UAAU,MAAoB;AAC5B,SAAK,UAAU,OAAO,QAAU,IAAI,KAAK;AACzC,SAAK;AACL,SAAK;AAEL,QAAI,KAAK,eAAe,GAAG;AACzB,WAAK,KAAK,KAAK,UAAU,CAAC;AAC1B,WAAK,QAAQ;AACb,WAAK,aAAa;AAAA,IACpB;AAAA,EACF;AAAA,EAEA,OAAO,OAAgD;AACrD,YAAQ,OAAO,OAAA;AAAA,MACb,KAAK,UAAU;AACb,aAAK,OAAO,aAAa;AACzB,cAAM,cAAc,MAAM;AAC1B,YAAI,CAAC,aAAa;AAChB;AAAA,QACF;AAEA,iBAAS,IAAI,GAAG,IAAI,YAAY,QAAQ,KAAK;AAC3C,gBAAM,OAAO,YAAY,WAAW,CAAC;AACrC,eAAK,UAAU,OAAO,GAAI;AAC1B,eAAK,UAAW,SAAS,IAAK,GAAI;AAAA,QACpC;AACA;AAAA,MACF;AAAA,MACA,KAAK;AACH,aAAK,OAAO,aAAa;AACzB,iBAAS,IAAI,GAAG,IAAI,MAAM,QAAQ,KAAK;AACrC,gBAAM,OAAO,MAAM,WAAW,CAAC;AAC/B,eAAK,UAAU,OAAO,GAAI;AAC1B,eAAK,UAAW,SAAS,IAAK,GAAI;AAAA,QACpC;AACA;AAAA,MACF,KAAK;AACH,WAAG,WAAW,GAAG,OAAO,IAAI;AAC5B,aAAK,UAAU,GAAG,CAAC,CAAE;AACrB,aAAK,UAAU,GAAG,CAAC,CAAE;AACrB,aAAK,UAAU,GAAG,CAAC,CAAE;AACrB,aAAK,UAAU,GAAG,CAAC,CAAE;AACrB,aAAK,UAAU,GAAG,CAAC,CAAE;AACrB,aAAK,UAAU,GAAG,CAAC,CAAE;AACrB,aAAK,UAAU,GAAG,CAAC,CAAE;AACrB,aAAK,UAAU,GAAG,CAAC,CAAE;AACrB;AAAA,MACF,KAAK,UAAU;AACb,YAAI,QAAQ;AACZ,YAAI,QAAQ,IAAI;AACd,kBAAQ,CAAC;AACT,eAAK,OAAO,kBAAkB;AAAA,QAChC,OAAO;AACL,eAAK,OAAO,cAAc;AAAA,QAC5B;AACA,eAAO,QAAQ,IAAI;AACjB,eAAK,UAAU,OAAO,QAAQ,KAAK,CAAC;AACpC,oBAAU;AAAA,QACZ;AACA,YAAI,UAAU,GAAI,MAAK,UAAU,CAAC;AAClC;AAAA,MACF;AAAA,MACA;AACE,cAAM,IAAI,UAAU,2BAA2B,OAAO,KAAK,EAAE;AAAA,IAAA;AAAA,EAEnE;AAAA,EAEA,SAAiB;AACf,QAAI,KAAK,aAAa,GAAG;AACvB,UAAI,KAAK,KAAK,UAAU;AACxB,WAAK,KAAK,KAAK,IAAI,UAAU;AAC7B,WAAM,MAAM,KAAO,OAAO;AAC1B,WAAK,KAAK,KAAK,IAAI,SAAU;AAC7B,WAAK,QAAQ;AAAA,IACf;AAEA,SAAK,QAAQ,KAAK;AAClB,SAAK,QAAQ,KAAK,SAAS;AAC3B,SAAK,OAAO,KAAK,KAAK,KAAK,MAAM,UAAU;AAC3C,SAAK,QAAQ,KAAK,SAAS;AAC3B,SAAK,OAAO,KAAK,KAAK,KAAK,MAAM,UAAU;AAC3C,SAAK,QAAQ,KAAK,SAAS;AAE3B,WAAO,KAAK,SAAS;AAAA,EACvB;AACF;;;"}
|
package/dist/esm/hashing/hash.js
CHANGED
|
@@ -10,6 +10,8 @@ const OBJECT_MARKER = randomHash();
|
|
|
10
10
|
const ARRAY_MARKER = randomHash();
|
|
11
11
|
const MAP_MARKER = randomHash();
|
|
12
12
|
const SET_MARKER = randomHash();
|
|
13
|
+
const UINT8ARRAY_MARKER = randomHash();
|
|
14
|
+
const UINT8ARRAY_CONTENT_HASH_THRESHOLD = 128;
|
|
13
15
|
const hashCache = /* @__PURE__ */ new WeakMap();
|
|
14
16
|
function hash(input) {
|
|
15
17
|
const hasher = new MurmurHashStream();
|
|
@@ -24,6 +26,17 @@ function hashObject(input) {
|
|
|
24
26
|
let valueHash;
|
|
25
27
|
if (input instanceof Date) {
|
|
26
28
|
valueHash = hashDate(input);
|
|
29
|
+
} else if (
|
|
30
|
+
// Check if input is a Uint8Array or Buffer
|
|
31
|
+
typeof Buffer !== `undefined` && input instanceof Buffer || input instanceof Uint8Array
|
|
32
|
+
) {
|
|
33
|
+
if (input.byteLength <= UINT8ARRAY_CONTENT_HASH_THRESHOLD) {
|
|
34
|
+
valueHash = hashUint8Array(input);
|
|
35
|
+
} else {
|
|
36
|
+
return cachedReferenceHash(input);
|
|
37
|
+
}
|
|
38
|
+
} else if (input instanceof File) {
|
|
39
|
+
return cachedReferenceHash(input);
|
|
27
40
|
} else {
|
|
28
41
|
let plainObjectInput = input;
|
|
29
42
|
let marker = OBJECT_MARKER;
|
|
@@ -38,9 +51,6 @@ function hashObject(input) {
|
|
|
38
51
|
marker = SET_MARKER;
|
|
39
52
|
plainObjectInput = [...input.entries()];
|
|
40
53
|
}
|
|
41
|
-
if (typeof Buffer !== `undefined` && input instanceof Buffer || input instanceof Uint8Array || input instanceof File) {
|
|
42
|
-
return cachedReferenceHash(input);
|
|
43
|
-
}
|
|
44
54
|
valueHash = hashPlainObject(plainObjectInput, marker);
|
|
45
55
|
}
|
|
46
56
|
hashCache.set(input, valueHash);
|
|
@@ -52,6 +62,15 @@ function hashDate(input) {
|
|
|
52
62
|
hasher.update(input.getTime());
|
|
53
63
|
return hasher.digest();
|
|
54
64
|
}
|
|
65
|
+
function hashUint8Array(input) {
|
|
66
|
+
const hasher = new MurmurHashStream();
|
|
67
|
+
hasher.update(UINT8ARRAY_MARKER);
|
|
68
|
+
hasher.update(input.byteLength);
|
|
69
|
+
for (let i = 0; i < input.byteLength; i++) {
|
|
70
|
+
hasher.writeByte(input[i]);
|
|
71
|
+
}
|
|
72
|
+
return hasher.digest();
|
|
73
|
+
}
|
|
55
74
|
function hashPlainObject(input, marker) {
|
|
56
75
|
const hasher = new MurmurHashStream();
|
|
57
76
|
hasher.update(marker);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"hash.js","sources":["../../../src/hashing/hash.ts"],"sourcesContent":["import { MurmurHashStream, randomHash } from \"./murmur.js\"\nimport type { Hasher } from \"./murmur.js\"\n\n/*\n * Implementation of structural hashing based on the Composites polyfill implementation:\n * https://github.com/tc39/proposal-composites\n */\n\nconst TRUE = randomHash()\nconst FALSE = randomHash()\nconst NULL = randomHash()\nconst UNDEFINED = randomHash()\nconst KEY = randomHash()\nconst FUNCTIONS = randomHash()\nconst DATE_MARKER = randomHash()\nconst OBJECT_MARKER = randomHash()\nconst ARRAY_MARKER = randomHash()\nconst MAP_MARKER = randomHash()\nconst SET_MARKER = randomHash()\n\nconst hashCache = new WeakMap<object, number>()\n\nexport function hash(input: any): number {\n const hasher = new MurmurHashStream()\n updateHasher(hasher, input)\n return hasher.digest()\n}\n\nfunction hashObject(input: object): number {\n const cachedHash = hashCache.get(input)\n if (cachedHash !== undefined) {\n return cachedHash\n }\n\n let valueHash: number | undefined\n if (input instanceof Date) {\n valueHash = hashDate(input)\n } else
|
|
1
|
+
{"version":3,"file":"hash.js","sources":["../../../src/hashing/hash.ts"],"sourcesContent":["import { MurmurHashStream, randomHash } from \"./murmur.js\"\nimport type { Hasher } from \"./murmur.js\"\n\n/*\n * Implementation of structural hashing based on the Composites polyfill implementation:\n * https://github.com/tc39/proposal-composites\n */\n\nconst TRUE = randomHash()\nconst FALSE = randomHash()\nconst NULL = randomHash()\nconst UNDEFINED = randomHash()\nconst KEY = randomHash()\nconst FUNCTIONS = randomHash()\nconst DATE_MARKER = randomHash()\nconst OBJECT_MARKER = randomHash()\nconst ARRAY_MARKER = randomHash()\nconst MAP_MARKER = randomHash()\nconst SET_MARKER = randomHash()\nconst UINT8ARRAY_MARKER = randomHash()\n\n// Maximum byte length for Uint8Arrays to hash by content instead of reference\n// Arrays smaller than this will be hashed by content, allowing proper equality comparisons\n// for small arrays like ULIDs (16 bytes) while still avoiding performance costs for large arrays\nconst UINT8ARRAY_CONTENT_HASH_THRESHOLD = 128\n\nconst hashCache = new WeakMap<object, number>()\n\nexport function hash(input: any): number {\n const hasher = new MurmurHashStream()\n updateHasher(hasher, input)\n return hasher.digest()\n}\n\nfunction hashObject(input: object): number {\n const cachedHash = hashCache.get(input)\n if (cachedHash !== undefined) {\n return cachedHash\n }\n\n let valueHash: number | undefined\n if (input instanceof Date) {\n valueHash = hashDate(input)\n } else if (\n // Check if input is a Uint8Array or Buffer\n (typeof Buffer !== `undefined` && input instanceof Buffer) ||\n input instanceof Uint8Array\n ) {\n // For small Uint8Arrays/Buffers (e.g., ULIDs, UUIDs), hash by content\n // to enable proper equality comparisons. For large arrays, hash by reference\n // to avoid performance costs.\n if (input.byteLength <= UINT8ARRAY_CONTENT_HASH_THRESHOLD) {\n valueHash = hashUint8Array(input)\n } else {\n // Deeply hashing large arrays would be too costly\n // so we track them by reference and cache them in a weak map\n return cachedReferenceHash(input)\n }\n } else if (input instanceof File) {\n // Files are always hashed by reference due to their potentially large size\n return cachedReferenceHash(input)\n } else {\n let plainObjectInput = input\n let marker = OBJECT_MARKER\n\n if (input instanceof Array) {\n marker = ARRAY_MARKER\n }\n\n if (input instanceof Map) {\n marker = MAP_MARKER\n plainObjectInput = [...input.entries()]\n }\n\n if (input instanceof Set) {\n marker = SET_MARKER\n plainObjectInput = [...input.entries()]\n }\n\n valueHash = hashPlainObject(plainObjectInput, marker)\n }\n\n hashCache.set(input, valueHash)\n return valueHash\n}\n\nfunction hashDate(input: Date): number {\n const hasher = new MurmurHashStream()\n hasher.update(DATE_MARKER)\n hasher.update(input.getTime())\n return hasher.digest()\n}\n\nfunction hashUint8Array(input: Uint8Array): number {\n const hasher = new MurmurHashStream()\n hasher.update(UINT8ARRAY_MARKER)\n // Hash the byte length first to differentiate arrays of different sizes\n hasher.update(input.byteLength)\n // Hash each byte in the array\n for (let i = 0; i < input.byteLength; i++) {\n hasher.writeByte(input[i]!)\n }\n return hasher.digest()\n}\n\nfunction hashPlainObject(input: object, marker: number): number {\n const hasher = new MurmurHashStream()\n\n // Mark the type of the input\n hasher.update(marker)\n const keys = Object.keys(input)\n keys.sort(keySort)\n for (const key of keys) {\n hasher.update(KEY)\n hasher.update(key)\n updateHasher(hasher, input[key as keyof typeof input])\n }\n\n return hasher.digest()\n}\n\nfunction updateHasher(hasher: Hasher, input: unknown): void {\n if (input === null) {\n hasher.update(NULL)\n return\n }\n switch (typeof input) {\n case `undefined`:\n hasher.update(UNDEFINED)\n return\n case `boolean`:\n hasher.update(input ? TRUE : FALSE)\n return\n case `number`:\n // Normalize NaNs and -0\n hasher.update(isNaN(input) ? NaN : input === 0 ? 0 : input)\n return\n case `bigint`:\n case `string`:\n case `symbol`:\n hasher.update(input)\n return\n case `object`:\n hasher.update(getCachedHash(input))\n return\n case `function`:\n // Functions are assigned a globally unique ID\n // and that ID is cached in the weak map\n hasher.update(cachedReferenceHash(input))\n return\n default:\n console.warn(\n `Ignored input during hashing because it is of type ${typeof input} which is not supported`\n )\n }\n}\n\nfunction getCachedHash(input: object): number {\n let valueHash = hashCache.get(input)\n if (valueHash === undefined) {\n valueHash = hashObject(input)\n }\n return valueHash\n}\n\nlet nextRefId = 1\nfunction cachedReferenceHash(fn: object): number {\n let valueHash = hashCache.get(fn)\n if (valueHash === undefined) {\n valueHash = nextRefId ^ FUNCTIONS\n nextRefId++\n hashCache.set(fn, valueHash)\n }\n return valueHash\n}\n\n/**\n * Strings sorted lexicographically.\n */\nfunction keySort(a: string, b: string): number {\n return a.localeCompare(b)\n}\n"],"names":[],"mappings":";AAQA,MAAM,OAAO,WAAA;AACb,MAAM,QAAQ,WAAA;AACd,MAAM,OAAO,WAAA;AACb,MAAM,YAAY,WAAA;AAClB,MAAM,MAAM,WAAA;AACZ,MAAM,YAAY,WAAA;AAClB,MAAM,cAAc,WAAA;AACpB,MAAM,gBAAgB,WAAA;AACtB,MAAM,eAAe,WAAA;AACrB,MAAM,aAAa,WAAA;AACnB,MAAM,aAAa,WAAA;AACnB,MAAM,oBAAoB,WAAA;AAK1B,MAAM,oCAAoC;AAE1C,MAAM,gCAAgB,QAAA;AAEf,SAAS,KAAK,OAAoB;AACvC,QAAM,SAAS,IAAI,iBAAA;AACnB,eAAa,QAAQ,KAAK;AAC1B,SAAO,OAAO,OAAA;AAChB;AAEA,SAAS,WAAW,OAAuB;AACzC,QAAM,aAAa,UAAU,IAAI,KAAK;AACtC,MAAI,eAAe,QAAW;AAC5B,WAAO;AAAA,EACT;AAEA,MAAI;AACJ,MAAI,iBAAiB,MAAM;AACzB,gBAAY,SAAS,KAAK;AAAA,EAC5B;AAAA;AAAA,IAEG,OAAO,WAAW,eAAe,iBAAiB,UACnD,iBAAiB;AAAA,IACjB;AAIA,QAAI,MAAM,cAAc,mCAAmC;AACzD,kBAAY,eAAe,KAAK;AAAA,IAClC,OAAO;AAGL,aAAO,oBAAoB,KAAK;AAAA,IAClC;AAAA,EACF,WAAW,iBAAiB,MAAM;AAEhC,WAAO,oBAAoB,KAAK;AAAA,EAClC,OAAO;AACL,QAAI,mBAAmB;AACvB,QAAI,SAAS;AAEb,QAAI,iBAAiB,OAAO;AAC1B,eAAS;AAAA,IACX;AAEA,QAAI,iBAAiB,KAAK;AACxB,eAAS;AACT,yBAAmB,CAAC,GAAG,MAAM,SAAS;AAAA,IACxC;AAEA,QAAI,iBAAiB,KAAK;AACxB,eAAS;AACT,yBAAmB,CAAC,GAAG,MAAM,SAAS;AAAA,IACxC;AAEA,gBAAY,gBAAgB,kBAAkB,MAAM;AAAA,EACtD;AAEA,YAAU,IAAI,OAAO,SAAS;AAC9B,SAAO;AACT;AAEA,SAAS,SAAS,OAAqB;AACrC,QAAM,SAAS,IAAI,iBAAA;AACnB,SAAO,OAAO,WAAW;AACzB,SAAO,OAAO,MAAM,SAAS;AAC7B,SAAO,OAAO,OAAA;AAChB;AAEA,SAAS,eAAe,OAA2B;AACjD,QAAM,SAAS,IAAI,iBAAA;AACnB,SAAO,OAAO,iBAAiB;AAE/B,SAAO,OAAO,MAAM,UAAU;AAE9B,WAAS,IAAI,GAAG,IAAI,MAAM,YAAY,KAAK;AACzC,WAAO,UAAU,MAAM,CAAC,CAAE;AAAA,EAC5B;AACA,SAAO,OAAO,OAAA;AAChB;AAEA,SAAS,gBAAgB,OAAe,QAAwB;AAC9D,QAAM,SAAS,IAAI,iBAAA;AAGnB,SAAO,OAAO,MAAM;AACpB,QAAM,OAAO,OAAO,KAAK,KAAK;AAC9B,OAAK,KAAK,OAAO;AACjB,aAAW,OAAO,MAAM;AACtB,WAAO,OAAO,GAAG;AACjB,WAAO,OAAO,GAAG;AACjB,iBAAa,QAAQ,MAAM,GAAyB,CAAC;AAAA,EACvD;AAEA,SAAO,OAAO,OAAA;AAChB;AAEA,SAAS,aAAa,QAAgB,OAAsB;AAC1D,MAAI,UAAU,MAAM;AAClB,WAAO,OAAO,IAAI;AAClB;AAAA,EACF;AACA,UAAQ,OAAO,OAAA;AAAA,IACb,KAAK;AACH,aAAO,OAAO,SAAS;AACvB;AAAA,IACF,KAAK;AACH,aAAO,OAAO,QAAQ,OAAO,KAAK;AAClC;AAAA,IACF,KAAK;AAEH,aAAO,OAAO,MAAM,KAAK,IAAI,MAAM,UAAU,IAAI,IAAI,KAAK;AAC1D;AAAA,IACF,KAAK;AAAA,IACL,KAAK;AAAA,IACL,KAAK;AACH,aAAO,OAAO,KAAK;AACnB;AAAA,IACF,KAAK;AACH,aAAO,OAAO,cAAc,KAAK,CAAC;AAClC;AAAA,IACF,KAAK;AAGH,aAAO,OAAO,oBAAoB,KAAK,CAAC;AACxC;AAAA,IACF;AACE,cAAQ;AAAA,QACN,sDAAsD,OAAO,KAAK;AAAA,MAAA;AAAA,EACpE;AAEN;AAEA,SAAS,cAAc,OAAuB;AAC5C,MAAI,YAAY,UAAU,IAAI,KAAK;AACnC,MAAI,cAAc,QAAW;AAC3B,gBAAY,WAAW,KAAK;AAAA,EAC9B;AACA,SAAO;AACT;AAEA,IAAI,YAAY;AAChB,SAAS,oBAAoB,IAAoB;AAC/C,MAAI,YAAY,UAAU,IAAI,EAAE;AAChC,MAAI,cAAc,QAAW;AAC3B,gBAAY,YAAY;AACxB;AACA,cAAU,IAAI,IAAI,SAAS;AAAA,EAC7B;AACA,SAAO;AACT;AAKA,SAAS,QAAQ,GAAW,GAAmB;AAC7C,SAAO,EAAE,cAAc,CAAC;AAC1B;"}
|
|
@@ -24,7 +24,7 @@ class MurmurHashStream {
|
|
|
24
24
|
this.hash = this.hash << 13 | this.hash >>> 19;
|
|
25
25
|
this.hash = Math.imul(this.hash, 5) + 3864292196;
|
|
26
26
|
}
|
|
27
|
-
|
|
27
|
+
writeByte(byte) {
|
|
28
28
|
this.carry |= (byte & 255) << 8 * this.carryBytes;
|
|
29
29
|
this.carryBytes++;
|
|
30
30
|
this.length++;
|
|
@@ -44,8 +44,8 @@ class MurmurHashStream {
|
|
|
44
44
|
}
|
|
45
45
|
for (let i = 0; i < description.length; i++) {
|
|
46
46
|
const code = description.charCodeAt(i);
|
|
47
|
-
this.
|
|
48
|
-
this.
|
|
47
|
+
this.writeByte(code & 255);
|
|
48
|
+
this.writeByte(code >>> 8 & 255);
|
|
49
49
|
}
|
|
50
50
|
return;
|
|
51
51
|
}
|
|
@@ -53,20 +53,20 @@ class MurmurHashStream {
|
|
|
53
53
|
this.update(STRING_MARKER);
|
|
54
54
|
for (let i = 0; i < chunk.length; i++) {
|
|
55
55
|
const code = chunk.charCodeAt(i);
|
|
56
|
-
this.
|
|
57
|
-
this.
|
|
56
|
+
this.writeByte(code & 255);
|
|
57
|
+
this.writeByte(code >>> 8 & 255);
|
|
58
58
|
}
|
|
59
59
|
return;
|
|
60
60
|
case `number`:
|
|
61
61
|
dv.setFloat64(0, chunk, true);
|
|
62
|
-
this.
|
|
63
|
-
this.
|
|
64
|
-
this.
|
|
65
|
-
this.
|
|
66
|
-
this.
|
|
67
|
-
this.
|
|
68
|
-
this.
|
|
69
|
-
this.
|
|
62
|
+
this.writeByte(u8[0]);
|
|
63
|
+
this.writeByte(u8[1]);
|
|
64
|
+
this.writeByte(u8[2]);
|
|
65
|
+
this.writeByte(u8[3]);
|
|
66
|
+
this.writeByte(u8[4]);
|
|
67
|
+
this.writeByte(u8[5]);
|
|
68
|
+
this.writeByte(u8[6]);
|
|
69
|
+
this.writeByte(u8[7]);
|
|
70
70
|
return;
|
|
71
71
|
case `bigint`: {
|
|
72
72
|
let value = chunk;
|
|
@@ -77,10 +77,10 @@ class MurmurHashStream {
|
|
|
77
77
|
this.update(BIG_INT_MARKER);
|
|
78
78
|
}
|
|
79
79
|
while (value > 0n) {
|
|
80
|
-
this.
|
|
80
|
+
this.writeByte(Number(value & 0xffn));
|
|
81
81
|
value >>= 8n;
|
|
82
82
|
}
|
|
83
|
-
if (chunk === 0n) this.
|
|
83
|
+
if (chunk === 0n) this.writeByte(0);
|
|
84
84
|
return;
|
|
85
85
|
}
|
|
86
86
|
default:
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"murmur.js","sources":["../../../src/hashing/murmur.ts"],"sourcesContent":["/*\n * Implementation of murmur hash based on the Composites polyfill implementation:\n * https://github.com/tc39/proposal-composites\n */\n\nconst RANDOM_SEED = randomHash()\nconst STRING_MARKER = randomHash()\nconst BIG_INT_MARKER = randomHash()\nconst NEG_BIG_INT_MARKER = randomHash()\nconst SYMBOL_MARKER = randomHash()\n\nexport type Hash = number\n\nexport function randomHash() {\n return (Math.random() * (2 ** 31 - 1)) >>> 0\n}\n\nexport interface Hasher {\n update: (val: symbol | string | number | bigint) => void\n digest: () => number\n}\n\n// Allocate these once, outside the hot path\nconst buf = new ArrayBuffer(8)\n// dv and u8 are 2 different views on the same buffer `buf`\nconst dv = new DataView(buf)\nconst u8 = new Uint8Array(buf)\n\n/**\n * This implementation of Murmur hash uses a random initial seed and random markers.\n * This means that hashes aren't deterministic across app restarts.\n * This is intentional in the composites polyfill to be resistent to hash-flooding attacks\n * where malicious users would precompute lots of different objects whose hashes collide with each other.\n *\n * Currently, for ts/db-ivm this is fine because we don't persist client state.\n * However, when we will introduce persistence we will either need to store the seeds or remove the randomness\n * to ensure deterministic hashes across app restarts.\n */\nexport class MurmurHashStream implements Hasher {\n private hash: number = RANDOM_SEED\n private length = 0\n private carry = 0\n private carryBytes = 0\n\n private _mix(k1: number): void {\n k1 = Math.imul(k1, 0xcc9e2d51)\n k1 = (k1 << 15) | (k1 >>> 17)\n k1 = Math.imul(k1, 0x1b873593)\n this.hash ^= k1\n this.hash = (this.hash << 13) | (this.hash >>> 19)\n this.hash = Math.imul(this.hash, 5) + 0xe6546b64\n }\n\n
|
|
1
|
+
{"version":3,"file":"murmur.js","sources":["../../../src/hashing/murmur.ts"],"sourcesContent":["/*\n * Implementation of murmur hash based on the Composites polyfill implementation:\n * https://github.com/tc39/proposal-composites\n */\n\nconst RANDOM_SEED = randomHash()\nconst STRING_MARKER = randomHash()\nconst BIG_INT_MARKER = randomHash()\nconst NEG_BIG_INT_MARKER = randomHash()\nconst SYMBOL_MARKER = randomHash()\n\nexport type Hash = number\n\nexport function randomHash() {\n return (Math.random() * (2 ** 31 - 1)) >>> 0\n}\n\nexport interface Hasher {\n update: (val: symbol | string | number | bigint) => void\n digest: () => number\n}\n\n// Allocate these once, outside the hot path\nconst buf = new ArrayBuffer(8)\n// dv and u8 are 2 different views on the same buffer `buf`\nconst dv = new DataView(buf)\nconst u8 = new Uint8Array(buf)\n\n/**\n * This implementation of Murmur hash uses a random initial seed and random markers.\n * This means that hashes aren't deterministic across app restarts.\n * This is intentional in the composites polyfill to be resistent to hash-flooding attacks\n * where malicious users would precompute lots of different objects whose hashes collide with each other.\n *\n * Currently, for ts/db-ivm this is fine because we don't persist client state.\n * However, when we will introduce persistence we will either need to store the seeds or remove the randomness\n * to ensure deterministic hashes across app restarts.\n */\nexport class MurmurHashStream implements Hasher {\n private hash: number = RANDOM_SEED\n private length = 0\n private carry = 0\n private carryBytes = 0\n\n private _mix(k1: number): void {\n k1 = Math.imul(k1, 0xcc9e2d51)\n k1 = (k1 << 15) | (k1 >>> 17)\n k1 = Math.imul(k1, 0x1b873593)\n this.hash ^= k1\n this.hash = (this.hash << 13) | (this.hash >>> 19)\n this.hash = Math.imul(this.hash, 5) + 0xe6546b64\n }\n\n writeByte(byte: number): void {\n this.carry |= (byte & 0xff) << (8 * this.carryBytes)\n this.carryBytes++\n this.length++\n\n if (this.carryBytes === 4) {\n this._mix(this.carry >>> 0)\n this.carry = 0\n this.carryBytes = 0\n }\n }\n\n update(chunk: symbol | string | number | bigint): void {\n switch (typeof chunk) {\n case `symbol`: {\n this.update(SYMBOL_MARKER)\n const description = chunk.description\n if (!description) {\n return\n }\n\n for (let i = 0; i < description.length; i++) {\n const code = description.charCodeAt(i)\n this.writeByte(code & 0xff)\n this.writeByte((code >>> 8) & 0xff)\n }\n return\n }\n case `string`:\n this.update(STRING_MARKER)\n for (let i = 0; i < chunk.length; i++) {\n const code = chunk.charCodeAt(i)\n this.writeByte(code & 0xff)\n this.writeByte((code >>> 8) & 0xff)\n }\n return\n case `number`:\n dv.setFloat64(0, chunk, true) // fixed little-endian\n this.writeByte(u8[0]!)\n this.writeByte(u8[1]!)\n this.writeByte(u8[2]!)\n this.writeByte(u8[3]!)\n this.writeByte(u8[4]!)\n this.writeByte(u8[5]!)\n this.writeByte(u8[6]!)\n this.writeByte(u8[7]!)\n return\n case `bigint`: {\n let value = chunk\n if (value < 0n) {\n value = -value\n this.update(NEG_BIG_INT_MARKER)\n } else {\n this.update(BIG_INT_MARKER)\n }\n while (value > 0n) {\n this.writeByte(Number(value & 0xffn))\n value >>= 8n\n }\n if (chunk === 0n) this.writeByte(0)\n return\n }\n default:\n throw new TypeError(`Unsupported input type: ${typeof chunk}`)\n }\n }\n\n digest(): number {\n if (this.carryBytes > 0) {\n let k1 = this.carry >>> 0\n k1 = Math.imul(k1, 0xcc9e2d51)\n k1 = (k1 << 15) | (k1 >>> 17)\n k1 = Math.imul(k1, 0x1b873593)\n this.hash ^= k1\n }\n\n this.hash ^= this.length\n this.hash ^= this.hash >>> 16\n this.hash = Math.imul(this.hash, 0x85ebca6b)\n this.hash ^= this.hash >>> 13\n this.hash = Math.imul(this.hash, 0xc2b2ae35)\n this.hash ^= this.hash >>> 16\n\n return this.hash >>> 0\n }\n}\n"],"names":[],"mappings":"AAKA,MAAM,cAAc,WAAA;AACpB,MAAM,gBAAgB,WAAA;AACtB,MAAM,iBAAiB,WAAA;AACvB,MAAM,qBAAqB,WAAA;AAC3B,MAAM,gBAAgB,WAAA;AAIf,SAAS,aAAa;AAC3B,SAAQ,KAAK,OAAA,KAAY,KAAK,KAAK,OAAQ;AAC7C;AAQA,MAAM,MAAM,IAAI,YAAY,CAAC;AAE7B,MAAM,KAAK,IAAI,SAAS,GAAG;AAC3B,MAAM,KAAK,IAAI,WAAW,GAAG;AAYtB,MAAM,iBAAmC;AAAA,EAAzC,cAAA;AACL,SAAQ,OAAe;AACvB,SAAQ,SAAS;AACjB,SAAQ,QAAQ;AAChB,SAAQ,aAAa;AAAA,EAAA;AAAA,EAEb,KAAK,IAAkB;AAC7B,SAAK,KAAK,KAAK,IAAI,UAAU;AAC7B,SAAM,MAAM,KAAO,OAAO;AAC1B,SAAK,KAAK,KAAK,IAAI,SAAU;AAC7B,SAAK,QAAQ;AACb,SAAK,OAAQ,KAAK,QAAQ,KAAO,KAAK,SAAS;AAC/C,SAAK,OAAO,KAAK,KAAK,KAAK,MAAM,CAAC,IAAI;AAAA,EACxC;AAAA,EAEA,UAAU,MAAoB;AAC5B,SAAK,UAAU,OAAO,QAAU,IAAI,KAAK;AACzC,SAAK;AACL,SAAK;AAEL,QAAI,KAAK,eAAe,GAAG;AACzB,WAAK,KAAK,KAAK,UAAU,CAAC;AAC1B,WAAK,QAAQ;AACb,WAAK,aAAa;AAAA,IACpB;AAAA,EACF;AAAA,EAEA,OAAO,OAAgD;AACrD,YAAQ,OAAO,OAAA;AAAA,MACb,KAAK,UAAU;AACb,aAAK,OAAO,aAAa;AACzB,cAAM,cAAc,MAAM;AAC1B,YAAI,CAAC,aAAa;AAChB;AAAA,QACF;AAEA,iBAAS,IAAI,GAAG,IAAI,YAAY,QAAQ,KAAK;AAC3C,gBAAM,OAAO,YAAY,WAAW,CAAC;AACrC,eAAK,UAAU,OAAO,GAAI;AAC1B,eAAK,UAAW,SAAS,IAAK,GAAI;AAAA,QACpC;AACA;AAAA,MACF;AAAA,MACA,KAAK;AACH,aAAK,OAAO,aAAa;AACzB,iBAAS,IAAI,GAAG,IAAI,MAAM,QAAQ,KAAK;AACrC,gBAAM,OAAO,MAAM,WAAW,CAAC;AAC/B,eAAK,UAAU,OAAO,GAAI;AAC1B,eAAK,UAAW,SAAS,IAAK,GAAI;AAAA,QACpC;AACA;AAAA,MACF,KAAK;AACH,WAAG,WAAW,GAAG,OAAO,IAAI;AAC5B,aAAK,UAAU,GAAG,CAAC,CAAE;AACrB,aAAK,UAAU,GAAG,CAAC,CAAE;AACrB,aAAK,UAAU,GAAG,CAAC,CAAE;AACrB,aAAK,UAAU,GAAG,CAAC,CAAE;AACrB,aAAK,UAAU,GAAG,CAAC,CAAE;AACrB,aAAK,UAAU,GAAG,CAAC,CAAE;AACrB,aAAK,UAAU,GAAG,CAAC,CAAE;AACrB,aAAK,UAAU,GAAG,CAAC,CAAE;AACrB;AAAA,MACF,KAAK,UAAU;AACb,YAAI,QAAQ;AACZ,YAAI,QAAQ,IAAI;AACd,kBAAQ,CAAC;AACT,eAAK,OAAO,kBAAkB;AAAA,QAChC,OAAO;AACL,eAAK,OAAO,cAAc;AAAA,QAC5B;AACA,eAAO,QAAQ,IAAI;AACjB,eAAK,UAAU,OAAO,QAAQ,KAAK,CAAC;AACpC,oBAAU;AAAA,QACZ;AACA,YAAI,UAAU,GAAI,MAAK,UAAU,CAAC;AAClC;AAAA,MACF;AAAA,MACA;AACE,cAAM,IAAI,UAAU,2BAA2B,OAAO,KAAK,EAAE;AAAA,IAAA;AAAA,EAEnE;AAAA,EAEA,SAAiB;AACf,QAAI,KAAK,aAAa,GAAG;AACvB,UAAI,KAAK,KAAK,UAAU;AACxB,WAAK,KAAK,KAAK,IAAI,UAAU;AAC7B,WAAM,MAAM,KAAO,OAAO;AAC1B,WAAK,KAAK,KAAK,IAAI,SAAU;AAC7B,WAAK,QAAQ;AAAA,IACf;AAEA,SAAK,QAAQ,KAAK;AAClB,SAAK,QAAQ,KAAK,SAAS;AAC3B,SAAK,OAAO,KAAK,KAAK,KAAK,MAAM,UAAU;AAC3C,SAAK,QAAQ,KAAK,SAAS;AAC3B,SAAK,OAAO,KAAK,KAAK,KAAK,MAAM,UAAU;AAC3C,SAAK,QAAQ,KAAK,SAAS;AAE3B,WAAO,KAAK,SAAS;AAAA,EACvB;AACF;"}
|
package/package.json
CHANGED
package/src/hashing/hash.ts
CHANGED
|
@@ -17,6 +17,12 @@ const OBJECT_MARKER = randomHash()
|
|
|
17
17
|
const ARRAY_MARKER = randomHash()
|
|
18
18
|
const MAP_MARKER = randomHash()
|
|
19
19
|
const SET_MARKER = randomHash()
|
|
20
|
+
const UINT8ARRAY_MARKER = randomHash()
|
|
21
|
+
|
|
22
|
+
// Maximum byte length for Uint8Arrays to hash by content instead of reference
|
|
23
|
+
// Arrays smaller than this will be hashed by content, allowing proper equality comparisons
|
|
24
|
+
// for small arrays like ULIDs (16 bytes) while still avoiding performance costs for large arrays
|
|
25
|
+
const UINT8ARRAY_CONTENT_HASH_THRESHOLD = 128
|
|
20
26
|
|
|
21
27
|
const hashCache = new WeakMap<object, number>()
|
|
22
28
|
|
|
@@ -35,6 +41,24 @@ function hashObject(input: object): number {
|
|
|
35
41
|
let valueHash: number | undefined
|
|
36
42
|
if (input instanceof Date) {
|
|
37
43
|
valueHash = hashDate(input)
|
|
44
|
+
} else if (
|
|
45
|
+
// Check if input is a Uint8Array or Buffer
|
|
46
|
+
(typeof Buffer !== `undefined` && input instanceof Buffer) ||
|
|
47
|
+
input instanceof Uint8Array
|
|
48
|
+
) {
|
|
49
|
+
// For small Uint8Arrays/Buffers (e.g., ULIDs, UUIDs), hash by content
|
|
50
|
+
// to enable proper equality comparisons. For large arrays, hash by reference
|
|
51
|
+
// to avoid performance costs.
|
|
52
|
+
if (input.byteLength <= UINT8ARRAY_CONTENT_HASH_THRESHOLD) {
|
|
53
|
+
valueHash = hashUint8Array(input)
|
|
54
|
+
} else {
|
|
55
|
+
// Deeply hashing large arrays would be too costly
|
|
56
|
+
// so we track them by reference and cache them in a weak map
|
|
57
|
+
return cachedReferenceHash(input)
|
|
58
|
+
}
|
|
59
|
+
} else if (input instanceof File) {
|
|
60
|
+
// Files are always hashed by reference due to their potentially large size
|
|
61
|
+
return cachedReferenceHash(input)
|
|
38
62
|
} else {
|
|
39
63
|
let plainObjectInput = input
|
|
40
64
|
let marker = OBJECT_MARKER
|
|
@@ -53,17 +77,6 @@ function hashObject(input: object): number {
|
|
|
53
77
|
plainObjectInput = [...input.entries()]
|
|
54
78
|
}
|
|
55
79
|
|
|
56
|
-
if (
|
|
57
|
-
(typeof Buffer !== `undefined` && input instanceof Buffer) ||
|
|
58
|
-
input instanceof Uint8Array ||
|
|
59
|
-
input instanceof File
|
|
60
|
-
) {
|
|
61
|
-
// Deeply hashing these objects would be too costly
|
|
62
|
-
// but we also don't want to ignore them
|
|
63
|
-
// so we track them by reference and cache them in a weak map
|
|
64
|
-
return cachedReferenceHash(input)
|
|
65
|
-
}
|
|
66
|
-
|
|
67
80
|
valueHash = hashPlainObject(plainObjectInput, marker)
|
|
68
81
|
}
|
|
69
82
|
|
|
@@ -78,6 +91,18 @@ function hashDate(input: Date): number {
|
|
|
78
91
|
return hasher.digest()
|
|
79
92
|
}
|
|
80
93
|
|
|
94
|
+
function hashUint8Array(input: Uint8Array): number {
|
|
95
|
+
const hasher = new MurmurHashStream()
|
|
96
|
+
hasher.update(UINT8ARRAY_MARKER)
|
|
97
|
+
// Hash the byte length first to differentiate arrays of different sizes
|
|
98
|
+
hasher.update(input.byteLength)
|
|
99
|
+
// Hash each byte in the array
|
|
100
|
+
for (let i = 0; i < input.byteLength; i++) {
|
|
101
|
+
hasher.writeByte(input[i]!)
|
|
102
|
+
}
|
|
103
|
+
return hasher.digest()
|
|
104
|
+
}
|
|
105
|
+
|
|
81
106
|
function hashPlainObject(input: object, marker: number): number {
|
|
82
107
|
const hasher = new MurmurHashStream()
|
|
83
108
|
|
package/src/hashing/murmur.ts
CHANGED
|
@@ -51,7 +51,7 @@ export class MurmurHashStream implements Hasher {
|
|
|
51
51
|
this.hash = Math.imul(this.hash, 5) + 0xe6546b64
|
|
52
52
|
}
|
|
53
53
|
|
|
54
|
-
|
|
54
|
+
writeByte(byte: number): void {
|
|
55
55
|
this.carry |= (byte & 0xff) << (8 * this.carryBytes)
|
|
56
56
|
this.carryBytes++
|
|
57
57
|
this.length++
|
|
@@ -74,8 +74,8 @@ export class MurmurHashStream implements Hasher {
|
|
|
74
74
|
|
|
75
75
|
for (let i = 0; i < description.length; i++) {
|
|
76
76
|
const code = description.charCodeAt(i)
|
|
77
|
-
this.
|
|
78
|
-
this.
|
|
77
|
+
this.writeByte(code & 0xff)
|
|
78
|
+
this.writeByte((code >>> 8) & 0xff)
|
|
79
79
|
}
|
|
80
80
|
return
|
|
81
81
|
}
|
|
@@ -83,20 +83,20 @@ export class MurmurHashStream implements Hasher {
|
|
|
83
83
|
this.update(STRING_MARKER)
|
|
84
84
|
for (let i = 0; i < chunk.length; i++) {
|
|
85
85
|
const code = chunk.charCodeAt(i)
|
|
86
|
-
this.
|
|
87
|
-
this.
|
|
86
|
+
this.writeByte(code & 0xff)
|
|
87
|
+
this.writeByte((code >>> 8) & 0xff)
|
|
88
88
|
}
|
|
89
89
|
return
|
|
90
90
|
case `number`:
|
|
91
91
|
dv.setFloat64(0, chunk, true) // fixed little-endian
|
|
92
|
-
this.
|
|
93
|
-
this.
|
|
94
|
-
this.
|
|
95
|
-
this.
|
|
96
|
-
this.
|
|
97
|
-
this.
|
|
98
|
-
this.
|
|
99
|
-
this.
|
|
92
|
+
this.writeByte(u8[0]!)
|
|
93
|
+
this.writeByte(u8[1]!)
|
|
94
|
+
this.writeByte(u8[2]!)
|
|
95
|
+
this.writeByte(u8[3]!)
|
|
96
|
+
this.writeByte(u8[4]!)
|
|
97
|
+
this.writeByte(u8[5]!)
|
|
98
|
+
this.writeByte(u8[6]!)
|
|
99
|
+
this.writeByte(u8[7]!)
|
|
100
100
|
return
|
|
101
101
|
case `bigint`: {
|
|
102
102
|
let value = chunk
|
|
@@ -107,10 +107,10 @@ export class MurmurHashStream implements Hasher {
|
|
|
107
107
|
this.update(BIG_INT_MARKER)
|
|
108
108
|
}
|
|
109
109
|
while (value > 0n) {
|
|
110
|
-
this.
|
|
110
|
+
this.writeByte(Number(value & 0xffn))
|
|
111
111
|
value >>= 8n
|
|
112
112
|
}
|
|
113
|
-
if (chunk === 0n) this.
|
|
113
|
+
if (chunk === 0n) this.writeByte(0)
|
|
114
114
|
return
|
|
115
115
|
}
|
|
116
116
|
default:
|