@functional-systems/memdrive 0.0.0-stage → 0.1.0-dev.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +13 -0
- package/LICENSE +201 -0
- package/README.md +229 -2
- package/dist/commonjs/chunking.d.ts +7 -0
- package/dist/commonjs/chunking.d.ts.map +1 -0
- package/dist/commonjs/chunking.js +46 -0
- package/dist/commonjs/chunking.js.map +1 -0
- package/dist/commonjs/errors.d.ts +17 -0
- package/dist/commonjs/errors.d.ts.map +1 -0
- package/dist/commonjs/errors.js +44 -0
- package/dist/commonjs/errors.js.map +1 -0
- package/dist/commonjs/filter.d.ts +30 -0
- package/dist/commonjs/filter.d.ts.map +1 -0
- package/dist/commonjs/filter.js +224 -0
- package/dist/commonjs/filter.js.map +1 -0
- package/dist/commonjs/index.d.ts +8 -0
- package/dist/commonjs/index.d.ts.map +1 -0
- package/dist/commonjs/index.js +343 -0
- package/dist/commonjs/index.js.map +1 -0
- package/dist/commonjs/package.json +3 -0
- package/dist/commonjs/storage.d.ts +274 -0
- package/dist/commonjs/storage.d.ts.map +1 -0
- package/dist/commonjs/storage.js +861 -0
- package/dist/commonjs/storage.js.map +1 -0
- package/dist/commonjs/tools/definitions.d.ts +5 -0
- package/dist/commonjs/tools/definitions.d.ts.map +1 -0
- package/dist/commonjs/tools/definitions.js +129 -0
- package/dist/commonjs/tools/definitions.js.map +1 -0
- package/dist/commonjs/tools/index.d.ts +6 -0
- package/dist/commonjs/tools/index.d.ts.map +1 -0
- package/dist/commonjs/tools/index.js +104 -0
- package/dist/commonjs/tools/index.js.map +1 -0
- package/dist/commonjs/tools/types.d.ts +50 -0
- package/dist/commonjs/tools/types.d.ts.map +1 -0
- package/dist/commonjs/tools/types.js +3 -0
- package/dist/commonjs/tools/types.js.map +1 -0
- package/dist/commonjs/types.d.ts +277 -0
- package/dist/commonjs/types.d.ts.map +1 -0
- package/dist/commonjs/types.js +3 -0
- package/dist/commonjs/types.js.map +1 -0
- package/dist/commonjs/values.d.ts +17 -0
- package/dist/commonjs/values.d.ts.map +1 -0
- package/dist/commonjs/values.js +90 -0
- package/dist/commonjs/values.js.map +1 -0
- package/dist/esm/chunking.d.ts +7 -0
- package/dist/esm/chunking.d.ts.map +1 -0
- package/dist/esm/chunking.js +42 -0
- package/dist/esm/chunking.js.map +1 -0
- package/dist/esm/errors.d.ts +17 -0
- package/dist/esm/errors.d.ts.map +1 -0
- package/dist/esm/errors.js +36 -0
- package/dist/esm/errors.js.map +1 -0
- package/dist/esm/filter.d.ts +30 -0
- package/dist/esm/filter.d.ts.map +1 -0
- package/dist/esm/filter.js +209 -0
- package/dist/esm/filter.js.map +1 -0
- package/dist/esm/index.d.ts +8 -0
- package/dist/esm/index.d.ts.map +1 -0
- package/dist/esm/index.js +335 -0
- package/dist/esm/index.js.map +1 -0
- package/dist/esm/package.json +3 -0
- package/dist/esm/storage.d.ts +274 -0
- package/dist/esm/storage.d.ts.map +1 -0
- package/dist/esm/storage.js +845 -0
- package/dist/esm/storage.js.map +1 -0
- package/dist/esm/tools/definitions.d.ts +5 -0
- package/dist/esm/tools/definitions.d.ts.map +1 -0
- package/dist/esm/tools/definitions.js +126 -0
- package/dist/esm/tools/definitions.js.map +1 -0
- package/dist/esm/tools/index.d.ts +6 -0
- package/dist/esm/tools/index.d.ts.map +1 -0
- package/dist/esm/tools/index.js +101 -0
- package/dist/esm/tools/index.js.map +1 -0
- package/dist/esm/tools/types.d.ts +50 -0
- package/dist/esm/tools/types.d.ts.map +1 -0
- package/dist/esm/tools/types.js +2 -0
- package/dist/esm/tools/types.js.map +1 -0
- package/dist/esm/types.d.ts +277 -0
- package/dist/esm/types.d.ts.map +1 -0
- package/dist/esm/types.js +2 -0
- package/dist/esm/types.js.map +1 -0
- package/dist/esm/values.d.ts +17 -0
- package/dist/esm/values.d.ts.map +1 -0
- package/dist/esm/values.js +87 -0
- package/dist/esm/values.js.map +1 -0
- package/docs/tools.md +163 -0
- package/package.json +100 -4
- package/src/chunking.ts +32 -0
- package/src/errors.ts +40 -0
- package/src/filter.ts +189 -0
- package/src/index.ts +323 -0
- package/src/storage.ts +765 -0
- package/src/tools/definitions.ts +156 -0
- package/src/tools/index.ts +97 -0
- package/src/tools/types.ts +45 -0
- package/src/types.ts +273 -0
- package/src/values.ts +103 -0
package/package.json
CHANGED
|
@@ -1,6 +1,102 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@functional-systems/memdrive",
|
|
3
|
-
"version": "0.
|
|
4
|
-
"
|
|
5
|
-
"
|
|
6
|
-
|
|
3
|
+
"version": "0.1.0-dev.5",
|
|
4
|
+
"description": "Source-linked memory storage and retrieval backed by LambdaDB.",
|
|
5
|
+
"type": "module",
|
|
6
|
+
"engines": {
|
|
7
|
+
"node": ">=20.12"
|
|
8
|
+
},
|
|
9
|
+
"sideEffects": false,
|
|
10
|
+
"files": [
|
|
11
|
+
"dist/",
|
|
12
|
+
"src/",
|
|
13
|
+
"README.md",
|
|
14
|
+
"CHANGELOG.md",
|
|
15
|
+
"LICENSE",
|
|
16
|
+
"NOTICE",
|
|
17
|
+
"docs/tools.md"
|
|
18
|
+
],
|
|
19
|
+
"main": "./dist/commonjs/index.js",
|
|
20
|
+
"module": "./dist/esm/index.js",
|
|
21
|
+
"types": "./dist/commonjs/index.d.ts",
|
|
22
|
+
"tshy": {
|
|
23
|
+
"exports": {
|
|
24
|
+
".": "./src/index.ts",
|
|
25
|
+
"./types": "./src/types.ts",
|
|
26
|
+
"./package.json": "./package.json",
|
|
27
|
+
"./tools": "./src/tools/index.ts"
|
|
28
|
+
}
|
|
29
|
+
},
|
|
30
|
+
"scripts": {
|
|
31
|
+
"lint": "eslint --max-warnings=0 src",
|
|
32
|
+
"typecheck": "tsc --noEmit",
|
|
33
|
+
"build": "tshy",
|
|
34
|
+
"test": "npm run build && npm run test:types && node --test test/*.test.mjs",
|
|
35
|
+
"test:types": "tsc -p test/types/tsconfig.json",
|
|
36
|
+
"check:version": "node scripts/version.mjs get",
|
|
37
|
+
"check:package": "npm run build && node scripts/check-package.mjs",
|
|
38
|
+
"prepublishOnly": "node scripts/check-publish.mjs",
|
|
39
|
+
"test:live": "npm run build && MEMDRIVE_RUN_LIVE_TESTS=1 node --env-file-if-exists=.env.local --test --test-concurrency=1 test/integration/*-live.test.mjs"
|
|
40
|
+
},
|
|
41
|
+
"dependencies": {
|
|
42
|
+
"@functional-systems/lambdadb": "0.12.0"
|
|
43
|
+
},
|
|
44
|
+
"devDependencies": {
|
|
45
|
+
"@eslint/js": "9.32.0",
|
|
46
|
+
"ajv": "8.20.0",
|
|
47
|
+
"esbuild": "0.28.2",
|
|
48
|
+
"eslint": "9.32.0",
|
|
49
|
+
"globals": "15.15.0",
|
|
50
|
+
"tshy": "2.0.1",
|
|
51
|
+
"typescript": "5.8.3",
|
|
52
|
+
"typescript-eslint": "8.38.0"
|
|
53
|
+
},
|
|
54
|
+
"license": "Apache-2.0",
|
|
55
|
+
"exports": {
|
|
56
|
+
".": {
|
|
57
|
+
"import": {
|
|
58
|
+
"types": "./dist/esm/index.d.ts",
|
|
59
|
+
"default": "./dist/esm/index.js"
|
|
60
|
+
},
|
|
61
|
+
"require": {
|
|
62
|
+
"types": "./dist/commonjs/index.d.ts",
|
|
63
|
+
"default": "./dist/commonjs/index.js"
|
|
64
|
+
}
|
|
65
|
+
},
|
|
66
|
+
"./types": {
|
|
67
|
+
"import": {
|
|
68
|
+
"types": "./dist/esm/types.d.ts",
|
|
69
|
+
"default": "./dist/esm/types.js"
|
|
70
|
+
},
|
|
71
|
+
"require": {
|
|
72
|
+
"types": "./dist/commonjs/types.d.ts",
|
|
73
|
+
"default": "./dist/commonjs/types.js"
|
|
74
|
+
}
|
|
75
|
+
},
|
|
76
|
+
"./package.json": "./package.json",
|
|
77
|
+
"./tools": {
|
|
78
|
+
"import": {
|
|
79
|
+
"types": "./dist/esm/tools/index.d.ts",
|
|
80
|
+
"default": "./dist/esm/tools/index.js"
|
|
81
|
+
},
|
|
82
|
+
"require": {
|
|
83
|
+
"types": "./dist/commonjs/tools/index.d.ts",
|
|
84
|
+
"default": "./dist/commonjs/tools/index.js"
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
},
|
|
88
|
+
"author": "Functional Systems, Inc.",
|
|
89
|
+
"homepage": "https://github.com/lambdadb/memdrive#readme",
|
|
90
|
+
"repository": {
|
|
91
|
+
"type": "git",
|
|
92
|
+
"url": "git+https://github.com/lambdadb/memdrive.git"
|
|
93
|
+
},
|
|
94
|
+
"bugs": {
|
|
95
|
+
"url": "https://github.com/lambdadb/memdrive/issues"
|
|
96
|
+
},
|
|
97
|
+
"publishConfig": {
|
|
98
|
+
"access": "public",
|
|
99
|
+
"tag": "dev"
|
|
100
|
+
},
|
|
101
|
+
"memdriveSourceCommit": "b7d498ab1c5779dc98f134b0744670889f21e963"
|
|
102
|
+
}
|
package/src/chunking.ts
ADDED
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
// Exact source partitions: UTF-16 offsets for slice(), a Unicode-code-point cap.
|
|
2
|
+
// English sentence detection is deliberate; other languages are not optimized.
|
|
3
|
+
export const CHUNKING_POLICY = 'boundary-v1';
|
|
4
|
+
export function chunkSpans(text: string, limit = 800): Array<{start: number; end: number; text: string}> {
|
|
5
|
+
if (!Number.isInteger(limit) || limit < 1) throw new RangeError('Chunk limit must be a positive integer.');
|
|
6
|
+
const offsets = [0];
|
|
7
|
+
for (const char of text) offsets.push(offsets.at(-1)! + char.length);
|
|
8
|
+
const paragraphs = [...text.matchAll(/\r?\n[\t ]*\r?\n(?:\s*\r?\n)*/gu)].map(m => m.index + m[0].length);
|
|
9
|
+
const sentences = typeof Intl.Segmenter === 'function'
|
|
10
|
+
? [...new Intl.Segmenter('en', { granularity: 'sentence' }).segment(text)].map(s => s.index + s.segment.length)
|
|
11
|
+
: [...text.matchAll(/[.!?]["'”’)\]]*(?:\s+|$)/gu)].map(m => m.index + m[0].length);
|
|
12
|
+
const words = [...text.matchAll(/\s+/gu)].map(m => m.index + m[0].length);
|
|
13
|
+
const lastBoundary = (boundaries: number[], floor: number, ceiling: number) => {
|
|
14
|
+
let low = 0, high = boundaries.length;
|
|
15
|
+
while (low < high) { const mid = (low + high) >>> 1; if (boundaries[mid]! <= ceiling) low = mid + 1; else high = mid; }
|
|
16
|
+
return boundaries[low - 1]! >= floor ? boundaries[low - 1] : undefined;
|
|
17
|
+
};
|
|
18
|
+
const spans: Array<{start: number; end: number; text: string}> = [];
|
|
19
|
+
for (let point = 0; point < offsets.length - 1;) {
|
|
20
|
+
const last = Math.min(point + limit, offsets.length - 1), start = offsets[point]!;
|
|
21
|
+
let end = offsets[last]!;
|
|
22
|
+
if (last < offsets.length - 1) {
|
|
23
|
+
// Avoid tiny fragments caused by an early heading/short sentence.
|
|
24
|
+
const floor = offsets[point + Math.max(1, Math.floor(limit / 2))]!;
|
|
25
|
+
end = lastBoundary(paragraphs, floor, end) ?? lastBoundary(sentences, floor, end)
|
|
26
|
+
?? lastBoundary(words, floor, end) ?? end;
|
|
27
|
+
}
|
|
28
|
+
spans.push({ start, end, text: text.slice(start, end) });
|
|
29
|
+
while (offsets[point]! < end) point++;
|
|
30
|
+
}
|
|
31
|
+
return spans;
|
|
32
|
+
}
|
package/src/errors.ts
ADDED
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
import type { RequestOptions, Usage } from './types.js';
|
|
2
|
+
|
|
3
|
+
export class MemDriveError extends Error {
|
|
4
|
+
declare code: string;
|
|
5
|
+
declare details?: Record<string, unknown> | undefined;
|
|
6
|
+
declare usage?: Usage | undefined;
|
|
7
|
+
declare status?: number | undefined;
|
|
8
|
+
constructor(code: string, message: string, { cause, details, usage }: { cause?: unknown; details?: Record<string, unknown> | undefined; usage?: Usage | undefined } = {}) {
|
|
9
|
+
super(message, cause === undefined ? undefined : { cause });
|
|
10
|
+
this.name = 'MemDriveError';
|
|
11
|
+
this.code = code;
|
|
12
|
+
if (details !== undefined) this.details = details;
|
|
13
|
+
if (usage !== undefined) this.usage = usage;
|
|
14
|
+
}
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
export function requireInput(condition: unknown, message: string): asserts condition {
|
|
18
|
+
if (!condition) throw new MemDriveError('INVALID_INPUT', message);
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
export function operationOptions(options: RequestOptions = {}): RequestOptions {
|
|
22
|
+
requireInput(options !== null && typeof options === 'object' && !Array.isArray(options)
|
|
23
|
+
&& Object.keys(options).every(key => key === 'signal'), 'Operation options accept only signal.');
|
|
24
|
+
const { signal } = options;
|
|
25
|
+
requireInput(signal === undefined || (signal !== null && typeof signal === 'object'
|
|
26
|
+
&& typeof signal.aborted === 'boolean' && typeof signal.addEventListener === 'function'
|
|
27
|
+
&& typeof signal.removeEventListener === 'function'), 'signal must be an AbortSignal.');
|
|
28
|
+
return { signal };
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
export function checkAbort(signal?: AbortSignal) {
|
|
32
|
+
if (signal?.aborted) throw new MemDriveError('ABORTED', 'Operation aborted.');
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
export function abortError(cause: unknown, signal?: AbortSignal): unknown {
|
|
36
|
+
const error = cause as { code?: string; name?: string } | undefined;
|
|
37
|
+
if (error?.code === 'ABORTED') return cause;
|
|
38
|
+
return signal?.aborted || error?.name === 'AbortError'
|
|
39
|
+
? new MemDriveError('ABORTED', 'Operation aborted.', { cause }) : cause;
|
|
40
|
+
}
|
package/src/filter.ts
ADDED
|
@@ -0,0 +1,189 @@
|
|
|
1
|
+
import type { Filter, FilterScalar, Metadata, MetadataSchema, MetadataType } from './types.js';
|
|
2
|
+
|
|
3
|
+
import { requireInput } from './errors.js';
|
|
4
|
+
|
|
5
|
+
export const isObject = (value: unknown): value is Record<string, unknown> => value !== null && typeof value === 'object'
|
|
6
|
+
&& !Array.isArray(value) && [Object.prototype, null].includes(Object.getPrototypeOf(value));
|
|
7
|
+
const own = (value: object, key: PropertyKey) => Object.hasOwn(value, key);
|
|
8
|
+
const kinds = new Set(['string', 'number', 'boolean', 'datetime', 'string[]']);
|
|
9
|
+
export const matchAll = () => ({ queryString: { query: '*:*' } });
|
|
10
|
+
|
|
11
|
+
// LambdaDB datetime indexes store epoch milliseconds. Reject finer precision
|
|
12
|
+
// instead of silently changing equality/range semantics; trailing zeroes are exact.
|
|
13
|
+
export function dateValue(value: unknown) {
|
|
14
|
+
if (typeof value !== 'string') return undefined;
|
|
15
|
+
const m = /^(\d{4})-(\d{2})-(\d{2})T(\d{2}):(\d{2}):(\d{2})(?:\.(\d{1,9}))?(Z|[+-]\d{2}:\d{2})$/.exec(value);
|
|
16
|
+
if (!m) return undefined;
|
|
17
|
+
const year = m[1]!, month = m[2]!, day = m[3]!, hour = m[4]!, minute = m[5]!, second = m[6]!, fraction = m[7] ?? '', zone = m[8]!;
|
|
18
|
+
if (/[1-9]/.test(fraction.slice(3))) return undefined;
|
|
19
|
+
const y = Number(year), mo = Number(month), d = Number(day);
|
|
20
|
+
const days = [31, y % 4 === 0 && (y % 100 !== 0 || y % 400 === 0) ? 29 : 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31];
|
|
21
|
+
if (mo < 1 || mo > 12 || d < 1 || d > days[mo - 1]! || Number(hour) > 23
|
|
22
|
+
|| Number(minute) > 59 || Number(second) > 59
|
|
23
|
+
|| (zone !== 'Z' && (Number(zone.slice(1, 3)) > 23 || Number(zone.slice(4)) > 59))) return undefined;
|
|
24
|
+
const millis = Date.parse(`${year}-${month}-${day}T${hour}:${minute}:${second}${zone}`);
|
|
25
|
+
return Number.isFinite(millis) ? BigInt(millis) + BigInt(fraction.padEnd(3, '0').slice(0, 3)) : undefined;
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
export function validateDescription(description: unknown): string | undefined {
|
|
29
|
+
requireInput(description === undefined || (typeof description === 'string' && description.length <= 255),
|
|
30
|
+
'Description must be text of at most 255 characters; use an empty string to clear it.');
|
|
31
|
+
return description;
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
export function validateMetadataSchema(schema: unknown): MetadataSchema {
|
|
35
|
+
requireInput(isObject(schema), 'metadataSchema must be an object.');
|
|
36
|
+
for (const [name, declaration] of Object.entries(schema)) {
|
|
37
|
+
requireInput(/^[A-Za-z_][A-Za-z0-9_]*$/.test(name)
|
|
38
|
+
&& !['id', '__proto__', 'prototype', 'constructor'].includes(name), 'Metadata field names must be unambiguous identifiers and cannot be reserved.');
|
|
39
|
+
requireInput(isObject(declaration) && Object.keys(declaration).every(key => ['type', 'description'].includes(key))
|
|
40
|
+
&& own(declaration, 'type') && typeof declaration.type === 'string' && kinds.has(declaration.type), 'Declare each metadata field with a supported type and optional description.');
|
|
41
|
+
validateDescription(declaration.description);
|
|
42
|
+
}
|
|
43
|
+
return structuredClone(schema) as MetadataSchema;
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
export function fieldType(field: unknown, schema: MetadataSchema): MetadataType | undefined {
|
|
47
|
+
if (field === 'id') return 'string';
|
|
48
|
+
if (field === 'timestamp') return 'datetime';
|
|
49
|
+
if (typeof field === 'string' && field.startsWith('metadata.')) {
|
|
50
|
+
const name = field.slice(9);
|
|
51
|
+
if (own(schema, name)) return schema[name]!.type;
|
|
52
|
+
}
|
|
53
|
+
return undefined;
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
export function validScalar(value: unknown, type: MetadataType | undefined) {
|
|
57
|
+
if (type === 'string' || type === 'string[]') return typeof value === 'string' && value.length <= 4096;
|
|
58
|
+
if (type === 'number') return typeof value === 'number' && Number.isFinite(value);
|
|
59
|
+
if (type === 'boolean') return typeof value === 'boolean';
|
|
60
|
+
return type === 'datetime' && dateValue(value) !== undefined;
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
export function validateFilter(filter: unknown, schema: MetadataSchema): Filter | undefined {
|
|
64
|
+
if (filter === undefined) return undefined;
|
|
65
|
+
const seen = new Set<object>();
|
|
66
|
+
const visit = (value: unknown): void => {
|
|
67
|
+
requireInput(isObject(value) && !seen.has(value), 'Filter must be an acyclic explicit Boolean expression.');
|
|
68
|
+
seen.add(value);
|
|
69
|
+
const keys = Object.keys(value);
|
|
70
|
+
if (keys.length === 1 && ['and', 'or'].includes(keys[0]!)) {
|
|
71
|
+
const children = value[keys[0]!];
|
|
72
|
+
requireInput(Array.isArray(children) && children.length > 0, 'and/or require nonempty arrays.');
|
|
73
|
+
children.forEach(visit);
|
|
74
|
+
} else {
|
|
75
|
+
requireInput(keys.length === 3 && ['field', 'op', 'value'].every(key => own(value, key)), 'Filter leaves require exactly field, op and value.');
|
|
76
|
+
const type = fieldType(value.field, schema);
|
|
77
|
+
requireInput(type !== undefined, 'Filter field must be id, timestamp, or declared metadata.');
|
|
78
|
+
if (value.op === 'exists') requireInput(typeof value.value === 'boolean', 'exists requires a boolean.');
|
|
79
|
+
else if (value.op === 'in') requireInput(Array.isArray(value.value) && value.value.length > 0
|
|
80
|
+
&& value.value.every(item => validScalar(item, type)), 'in requires a nonempty list of correctly typed scalar values.');
|
|
81
|
+
else {
|
|
82
|
+
requireInput(typeof value.op === 'string' && ['eq', 'neq', 'gt', 'gte', 'lt', 'lte'].includes(value.op), 'Unsupported filter operator.');
|
|
83
|
+
requireInput(validScalar(value.value, type), 'Filter value does not match the declared field type.');
|
|
84
|
+
requireInput(['eq', 'neq'].includes(value.op) || ['number', 'datetime'].includes(type), 'Ordering is supported only for numbers and datetimes.');
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
seen.delete(value);
|
|
88
|
+
};
|
|
89
|
+
visit(filter);
|
|
90
|
+
return structuredClone(filter) as Filter;
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
export interface NativeFilter { queryString?: {query: string; defaultField?: string}; bool?: NativeFilter[]; occur?: string | undefined }
|
|
94
|
+
|
|
95
|
+
const queryString = (field: string, query: string) => ({ queryString: { query, defaultField: field } });
|
|
96
|
+
// Quote even literal *:*: LambdaDB treats that exact string as match-all with skipSyntax.
|
|
97
|
+
const quote = (value: unknown) => `"${String(value).replace(/[\\"]/g, '\\$&')}"`;
|
|
98
|
+
const bool = (clauses: NativeFilter[], occur: string): NativeFilter => ({ bool: clauses.map(clause => ({ ...clause, occur })) });
|
|
99
|
+
function exists(field: string, type: MetadataType | undefined): NativeFilter {
|
|
100
|
+
if (type === 'boolean') return bool([queryString(field, 'true'), queryString(field, 'false')], 'should');
|
|
101
|
+
// A bare wildcard is rejected by the native parser's leading-wildcard policy.
|
|
102
|
+
return queryString(field, '[* TO *]');
|
|
103
|
+
}
|
|
104
|
+
function decimal(value: FilterScalar) {
|
|
105
|
+
const text = String(value);
|
|
106
|
+
if (!text.includes('e')) return text;
|
|
107
|
+
// Native numeric queries use DecimalFormat, which does not parse JS's
|
|
108
|
+
// lowercase exponent syntax. Expand the shortest round-trippable decimal.
|
|
109
|
+
const [mantissa, exponent] = text.split('e') as [string, string];
|
|
110
|
+
const sign = mantissa.startsWith('-') ? '-' : '';
|
|
111
|
+
const unsigned = sign ? mantissa.slice(1) : mantissa;
|
|
112
|
+
const point = (unsigned.indexOf('.') === -1 ? unsigned.length : unsigned.indexOf('.')) + Number(exponent);
|
|
113
|
+
const digits = unsigned.replace('.', '');
|
|
114
|
+
return sign + (point <= 0 ? `0.${'0'.repeat(-point)}${digits}`
|
|
115
|
+
: point >= digits.length ? `${digits}${'0'.repeat(point - digits.length)}`
|
|
116
|
+
: `${digits.slice(0, point)}.${digits.slice(point)}`);
|
|
117
|
+
}
|
|
118
|
+
function equal(field: string, value: FilterScalar, type: MetadataType | undefined): NativeFilter {
|
|
119
|
+
const literal = type === 'number' ? decimal(value) : type === 'boolean' ? String(value) : quote(value);
|
|
120
|
+
return queryString(field, type === 'number' || type === 'datetime' ? `[${literal} TO ${literal}]` : literal);
|
|
121
|
+
}
|
|
122
|
+
function nativeFilter(filter: Filter | undefined, schema: MetadataSchema): NativeFilter {
|
|
123
|
+
if (filter === undefined) return matchAll();
|
|
124
|
+
if (filter.and) return bool(filter.and.map(item => nativeFilter(item, schema)), 'filter');
|
|
125
|
+
if (filter.or) return bool(filter.or.map(item => nativeFilter(item, schema)), 'should');
|
|
126
|
+
const { op, value } = filter, type = fieldType(filter.field, schema);
|
|
127
|
+
const field = filter.field === 'id' ? 'memId' : filter.field;
|
|
128
|
+
if (op === 'exists') return value ? exists(field, type) : { bool: [
|
|
129
|
+
{ ...matchAll(), occur: 'filter' }, { ...exists(field, type), occur: 'must_not' },
|
|
130
|
+
] };
|
|
131
|
+
if (op === 'eq') return equal(field, value, type);
|
|
132
|
+
if (op === 'neq') return { bool: [
|
|
133
|
+
{ ...exists(field, type), occur: 'filter' }, { ...equal(field, value, type), occur: 'must_not' },
|
|
134
|
+
] };
|
|
135
|
+
if (op === 'in') return bool(value.map(item => equal(field, item, type)), 'should');
|
|
136
|
+
const literal = type === 'datetime' ? quote(value) : decimal(value);
|
|
137
|
+
const inclusive = ['gte', 'lte'].includes(op);
|
|
138
|
+
return queryString(field, `${inclusive ? '[' : '{'}${op.startsWith('g') ? `${literal} TO *` : `* TO ${literal}`}${inclusive ? ']' : '}'}`);
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
export function compileFilter(filter: unknown, schema: MetadataSchema): NativeFilter {
|
|
142
|
+
return nativeFilter(validateFilter(filter, schema), schema);
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
export function buildFilter(filters: unknown[] = [], excludeIds: string[] = [], schema: MetadataSchema = {}): NativeFilter {
|
|
146
|
+
requireInput(Array.isArray(filters) && Array.isArray(excludeIds)
|
|
147
|
+
&& excludeIds.every(id => typeof id === 'string' && id.length > 0 && id.length <= 512), 'Invalid filters or excluded IDs.');
|
|
148
|
+
const clauses = filters.filter(item => item !== undefined).map(item => ({ ...compileFilter(item, schema), occur: 'filter' }));
|
|
149
|
+
if (clauses.length === 0) clauses.push({ ...matchAll(), occur: 'filter' });
|
|
150
|
+
clauses.push(...excludeIds.map(id => ({ ...equal('memId', id, 'string'), occur: 'must_not' })));
|
|
151
|
+
if (clauses.length === 1) { const { occur: _occur, ...clause } = clauses[0]!; return clause; }
|
|
152
|
+
return { bool: clauses };
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
export function buildSearchFilter(filters: unknown[] = [], excludeIds: string[] = [], seenChunkIds: string[] = [], schema: MetadataSchema = {}): NativeFilter {
|
|
156
|
+
requireInput(Array.isArray(seenChunkIds) && seenChunkIds.every(id => typeof id === 'string' && id.length > 0),
|
|
157
|
+
'Invalid seen chunk IDs.');
|
|
158
|
+
return { bool: [
|
|
159
|
+
{ ...buildFilter(filters, excludeIds, schema), occur: 'filter' },
|
|
160
|
+
...seenChunkIds.map(id => ({ ...equal('id', id, 'string'), occur: 'must_not' })),
|
|
161
|
+
] };
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
export function matchesFilter(mem: {id: string; timestamp?: string | undefined; metadata?: Metadata | undefined}, filter: Filter | undefined, schema: MetadataSchema): boolean {
|
|
165
|
+
validateFilter(filter, schema);
|
|
166
|
+
const visit = (node: Filter | undefined): boolean => {
|
|
167
|
+
if (node === undefined) return true;
|
|
168
|
+
if (node.and) return node.and.every(visit);
|
|
169
|
+
if (node.or) return node.or.some(visit);
|
|
170
|
+
const { field, op, value } = node;
|
|
171
|
+
const stored = field.startsWith('metadata.') ? mem.metadata?.[field.slice(9)] : mem[field as 'id' | 'timestamp'];
|
|
172
|
+
const values = stored === undefined ? [] : Array.isArray(stored) ? stored : [stored];
|
|
173
|
+
if (op === 'exists') return (values.length > 0) === value;
|
|
174
|
+
const type = fieldType(field, schema);
|
|
175
|
+
const convert = (item: unknown) => type === 'datetime' ? dateValue(item) : item;
|
|
176
|
+
const same = (expected: FilterScalar) => values.some(item => convert(item) === convert(expected));
|
|
177
|
+
if (op === 'eq') return same(value);
|
|
178
|
+
if (op === 'neq') return values.length > 0 && !same(value);
|
|
179
|
+
if (op === 'in') return value.some(same);
|
|
180
|
+
return values.some(item => {
|
|
181
|
+
const a = convert(item) as number | bigint, b = convert(value) as number | bigint;
|
|
182
|
+
if (op === 'gt') return a > b;
|
|
183
|
+
if (op === 'gte') return a >= b;
|
|
184
|
+
if (op === 'lt') return a < b;
|
|
185
|
+
return a <= b;
|
|
186
|
+
});
|
|
187
|
+
};
|
|
188
|
+
return visit(filter);
|
|
189
|
+
}
|