@dxos/index-core 0.10.0 → 0.11.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/lib/index.mjs +725 -0
- package/dist/lib/index.mjs.map +1 -0
- package/dist/types/src/indexes/entity-meta-index.d.ts +1 -1
- package/dist/types/src/indexes/entity-meta-index.d.ts.map +1 -1
- package/dist/types/src/indexes/fts-index.d.ts.map +1 -1
- package/dist/types/tsconfig.tsbuildinfo +1 -1
- package/package.json +8 -8
- package/src/indexes/entity-meta-index.test.ts +41 -0
- package/src/indexes/entity-meta-index.ts +78 -22
- package/src/indexes/fts-index.ts +21 -3
- package/dist/lib/neutral/index.mjs +0 -837
- package/dist/lib/neutral/index.mjs.map +0 -7
- package/dist/lib/neutral/meta.json +0 -1
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@dxos/index-core",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.11.1",
|
|
4
4
|
"description": "Indexing core.",
|
|
5
5
|
"homepage": "https://dxos.org",
|
|
6
6
|
"bugs": "https://github.com/dxos/dxos/issues",
|
|
@@ -16,7 +16,7 @@
|
|
|
16
16
|
".": {
|
|
17
17
|
"source": "./src/index.ts",
|
|
18
18
|
"types": "./dist/types/src/index.d.ts",
|
|
19
|
-
"
|
|
19
|
+
"import": "./dist/lib/index.mjs"
|
|
20
20
|
}
|
|
21
21
|
},
|
|
22
22
|
"files": [
|
|
@@ -28,12 +28,12 @@
|
|
|
28
28
|
"@effect/platform": "0.96.2",
|
|
29
29
|
"@effect/sql": "0.51.1",
|
|
30
30
|
"effect": "3.21.4",
|
|
31
|
-
"@dxos/
|
|
32
|
-
"@dxos/
|
|
33
|
-
"@dxos/
|
|
34
|
-
"@dxos/invariant": "0.
|
|
35
|
-
"@dxos/
|
|
36
|
-
"@dxos/
|
|
31
|
+
"@dxos/context": "0.11.1",
|
|
32
|
+
"@dxos/echo-protocol": "0.11.1",
|
|
33
|
+
"@dxos/keys": "0.11.1",
|
|
34
|
+
"@dxos/invariant": "0.11.1",
|
|
35
|
+
"@dxos/sql-sqlite": "0.11.1",
|
|
36
|
+
"@dxos/echo": "0.11.1"
|
|
37
37
|
},
|
|
38
38
|
"devDependencies": {
|
|
39
39
|
"@effect/sql-sqlite-node": "0.52.0"
|
|
@@ -4,6 +4,7 @@
|
|
|
4
4
|
|
|
5
5
|
import * as Reactivity from '@effect/experimental/Reactivity';
|
|
6
6
|
import * as SqliteClient from '@effect/sql-sqlite-node/SqliteClient';
|
|
7
|
+
import * as SqlClient from '@effect/sql/SqlClient';
|
|
7
8
|
import { describe, expect, it } from '@effect/vitest';
|
|
8
9
|
import * as Effect from 'effect/Effect';
|
|
9
10
|
import * as Layer from 'effect/Layer';
|
|
@@ -66,6 +67,46 @@ describe('EntityMetaIndex', () => {
|
|
|
66
67
|
}).pipe(Effect.provide(TestLayer)),
|
|
67
68
|
);
|
|
68
69
|
|
|
70
|
+
it.effect('resolves a legacy single-slash type-identifier row when queried by the canonical form', () =>
|
|
71
|
+
Effect.gen(function* () {
|
|
72
|
+
const index = new EntityMetaIndex();
|
|
73
|
+
yield* index.migrate();
|
|
74
|
+
const sql = yield* SqlClient.SqlClient;
|
|
75
|
+
|
|
76
|
+
const spaceId = SpaceId.random();
|
|
77
|
+
const objectId = EntityId.random();
|
|
78
|
+
const schemaId = EntityId.random();
|
|
79
|
+
|
|
80
|
+
const item: IndexerObject = {
|
|
81
|
+
spaceId,
|
|
82
|
+
queueId: EntityId.random(),
|
|
83
|
+
queueNamespace: 'data',
|
|
84
|
+
documentId: null,
|
|
85
|
+
recordId: null,
|
|
86
|
+
createdAt: null,
|
|
87
|
+
updatedAt: Date.now(),
|
|
88
|
+
data: {
|
|
89
|
+
id: objectId,
|
|
90
|
+
[ATTR_TYPE]: EID.make({ entityId: schemaId }),
|
|
91
|
+
[ATTR_DELETED]: false,
|
|
92
|
+
},
|
|
93
|
+
};
|
|
94
|
+
|
|
95
|
+
yield* index.update([item]);
|
|
96
|
+
|
|
97
|
+
// The write path normalizes identifiers, so rewrite the row to the legacy single-slash form to
|
|
98
|
+
// simulate a pre-normalization on-disk row (parameterized to avoid SQL injection).
|
|
99
|
+
const legacyTypeDxn = `echo:/${schemaId}`;
|
|
100
|
+
yield* sql`UPDATE objectMeta SET typeDXN = ${legacyTypeDxn} WHERE objectId = ${objectId}`;
|
|
101
|
+
const [stored] = yield* sql<{ typeDXN: string }>`SELECT typeDXN FROM objectMeta WHERE objectId = ${objectId}`;
|
|
102
|
+
expect(stored.typeDXN).toBe(legacyTypeDxn);
|
|
103
|
+
|
|
104
|
+
// A canonical-form query must still resolve the legacy row without a reindex.
|
|
105
|
+
const results = yield* index.query({ spaceId, typeDXN: EID.make({ entityId: schemaId }) });
|
|
106
|
+
expect(results.map((meta) => meta.objectId)).toEqual([objectId]);
|
|
107
|
+
}).pipe(Effect.provide(TestLayer)),
|
|
108
|
+
);
|
|
109
|
+
|
|
69
110
|
it.effect('should not treat LIKE wildcards in versionless type queries', () =>
|
|
70
111
|
Effect.gen(function* () {
|
|
71
112
|
const index = new EntityMetaIndex();
|
|
@@ -16,8 +16,8 @@ import type { Index } from './interface';
|
|
|
16
16
|
|
|
17
17
|
/**
|
|
18
18
|
* Normalizes an echo: EID to the local (unqualified) form so SQL comparisons are consistent.
|
|
19
|
-
*
|
|
20
|
-
* otherwise miss every row for that type.
|
|
19
|
+
* Rows are indexed in the canonical `echo:///<entityId>` form; a space-qualified
|
|
20
|
+
* `echo://<space>/<entityId>` would otherwise miss every row for that type.
|
|
21
21
|
*/
|
|
22
22
|
const _normalizeTypeUri = (typeDXN: string): string => {
|
|
23
23
|
if (!typeDXN.startsWith('echo:')) {
|
|
@@ -31,6 +31,18 @@ const _normalizeTypeUri = (typeDXN: string): string => {
|
|
|
31
31
|
return entityId ? EID.make({ entityId }) : typeDXN;
|
|
32
32
|
};
|
|
33
33
|
|
|
34
|
+
/**
|
|
35
|
+
* Every stored `typeDXN` form equivalent to a normalized type identifier. Rows written before
|
|
36
|
+
* `echo:///<id>` became the canonical local EID hold the single-slash `echo:/<id>` form; both
|
|
37
|
+
* address the same stored schema, so queries must match either without requiring a reindex.
|
|
38
|
+
*/
|
|
39
|
+
const _typeUriEquivalents = (normalized: string): string[] => {
|
|
40
|
+
if (normalized.startsWith('echo:///')) {
|
|
41
|
+
return [normalized, `echo:/${normalized.slice('echo:///'.length)}`];
|
|
42
|
+
}
|
|
43
|
+
return [normalized];
|
|
44
|
+
};
|
|
45
|
+
|
|
34
46
|
const _escapeLikePrefix = (prefix: string) => {
|
|
35
47
|
// Escape LIKE metacharacters in the *literal* prefix (we still append a wildcard for the version suffix).
|
|
36
48
|
// Backslash is used as the ESCAPE character.
|
|
@@ -150,13 +162,16 @@ export class EntityMetaIndex implements Index {
|
|
|
150
162
|
const normalizedTypeDXN = _normalizeTypeUri(query.typeDXN);
|
|
151
163
|
const parsedDxn = DXN.isDXN(normalizedTypeDXN) ? normalizedTypeDXN : undefined;
|
|
152
164
|
const hasNoVersion = parsedDxn !== undefined && DXN.getVersion(parsedDxn) === undefined;
|
|
165
|
+
const forms = _typeUriEquivalents(normalizedTypeDXN);
|
|
166
|
+
const exactMatch = sql.or(forms.map((form) => sql`typeDXN = ${form}`));
|
|
167
|
+
// Version wildcard must cover every equivalent form so a versionless query still matches
|
|
168
|
+
// legacy versioned rows written under the single-slash prefix.
|
|
169
|
+
const likeMatch = sql.or(forms.map((form) => sql`typeDXN LIKE ${_escapeLikePrefix(form)} ESCAPE '\\'`));
|
|
153
170
|
|
|
154
171
|
// SQLite stores booleans as integers, so we need to specify the raw row type.
|
|
155
172
|
const rows = hasNoVersion
|
|
156
|
-
? yield* sql<EntityMeta>`SELECT * FROM objectMeta WHERE spaceId = ${query.spaceId} AND (
|
|
157
|
-
|
|
158
|
-
} OR typeDXN LIKE ${_escapeLikePrefix(normalizedTypeDXN)} ESCAPE '\\')`
|
|
159
|
-
: yield* sql<EntityMeta>`SELECT * FROM objectMeta WHERE spaceId = ${query.spaceId} AND typeDXN = ${normalizedTypeDXN}`;
|
|
173
|
+
? yield* sql<EntityMeta>`SELECT * FROM objectMeta WHERE spaceId = ${query.spaceId} AND (${exactMatch} OR ${likeMatch})`
|
|
174
|
+
: yield* sql<EntityMeta>`SELECT * FROM objectMeta WHERE spaceId = ${query.spaceId} AND ${exactMatch}`;
|
|
160
175
|
return rows.map((row) => ({
|
|
161
176
|
...row,
|
|
162
177
|
deleted: !!row.deleted,
|
|
@@ -229,9 +244,13 @@ export class EntityMetaIndex implements Index {
|
|
|
229
244
|
const normalized = _normalizeTypeUri(typeDXN);
|
|
230
245
|
const parsedDxn = DXN.isDXN(normalized) ? normalized : undefined;
|
|
231
246
|
const hasNoVersion = parsedDxn !== undefined && DXN.getVersion(parsedDxn) === undefined;
|
|
232
|
-
const
|
|
247
|
+
const forms = _typeUriEquivalents(normalized);
|
|
248
|
+
const exactMatch = sql.or(forms.map((form) => sql`typeDXN = ${form}`));
|
|
233
249
|
return hasNoVersion
|
|
234
|
-
? sql.or([
|
|
250
|
+
? sql.or([
|
|
251
|
+
exactMatch,
|
|
252
|
+
sql.or(forms.map((form) => sql`typeDXN LIKE ${_escapeLikePrefix(form)} ESCAPE '\\'`)),
|
|
253
|
+
])
|
|
235
254
|
: exactMatch;
|
|
236
255
|
}),
|
|
237
256
|
);
|
|
@@ -287,15 +306,21 @@ export class EntityMetaIndex implements Index {
|
|
|
287
306
|
const objectId = castData.id;
|
|
288
307
|
|
|
289
308
|
// Check for existing record by (spaceId, queueId) or (spaceId, documentId).
|
|
290
|
-
|
|
309
|
+
type ExistingRow = {
|
|
310
|
+
recordId: number;
|
|
311
|
+
entityKind: string;
|
|
312
|
+
typeDXN: string;
|
|
313
|
+
source: string | null;
|
|
314
|
+
target: string | null;
|
|
315
|
+
parent: string | null;
|
|
316
|
+
};
|
|
317
|
+
let existing: readonly ExistingRow[];
|
|
291
318
|
if (documentId) {
|
|
292
|
-
existing =
|
|
293
|
-
recordId
|
|
294
|
-
}>`SELECT recordId FROM objectMeta WHERE spaceId = ${spaceId} AND documentId = ${documentId} AND objectId = ${objectId} LIMIT 1`;
|
|
319
|
+
existing =
|
|
320
|
+
yield* sql<ExistingRow>`SELECT recordId, entityKind, typeDXN, source, target, parent FROM objectMeta WHERE spaceId = ${spaceId} AND documentId = ${documentId} AND objectId = ${objectId} LIMIT 1`;
|
|
295
321
|
} else if (queueId) {
|
|
296
|
-
existing =
|
|
297
|
-
recordId
|
|
298
|
-
}>`SELECT recordId FROM objectMeta WHERE spaceId = ${spaceId} AND queueId = ${queueId} AND objectId = ${objectId} LIMIT 1`;
|
|
322
|
+
existing =
|
|
323
|
+
yield* sql<ExistingRow>`SELECT recordId, entityKind, typeDXN, source, target, parent FROM objectMeta WHERE spaceId = ${spaceId} AND queueId = ${queueId} AND objectId = ${objectId} LIMIT 1`;
|
|
299
324
|
} else {
|
|
300
325
|
// Should not happen based on IndexerObject definition (one must be present ideally), but handle gracefully.
|
|
301
326
|
existing = [];
|
|
@@ -306,17 +331,44 @@ export class EntityMetaIndex implements Index {
|
|
|
306
331
|
const [{ v }] = result;
|
|
307
332
|
const version = (v ?? 0) + 1;
|
|
308
333
|
|
|
334
|
+
// A partial block carries no `@type`/body — notably the `{ id, '@deleted': true }`
|
|
335
|
+
// tombstone appended by `Feed.remove`. Preserve the prior row's body-derived columns
|
|
336
|
+
// (type, kind, relation endpoints, parent) rather than recomputing them from the empty
|
|
337
|
+
// block; otherwise `typeDXN` collapses to the `'type'` fallback and type-scoped queries
|
|
338
|
+
// with `deleted: 'include'` stop matching the deleted object. Only `deleted`/`version`/
|
|
339
|
+
// `updatedAt` advance for such a block.
|
|
340
|
+
const priorRow = existing.length > 0 ? existing[0] : undefined;
|
|
341
|
+
const isPartialBlock = castData[ATTR_TYPE] === undefined;
|
|
342
|
+
const preserveBody = isPartialBlock && priorRow !== undefined;
|
|
343
|
+
|
|
309
344
|
// Extract metadata.
|
|
310
|
-
const entityKind =
|
|
345
|
+
const entityKind = preserveBody
|
|
346
|
+
? priorRow.entityKind
|
|
347
|
+
: castData[ATTR_RELATION_SOURCE]
|
|
348
|
+
? 'relation'
|
|
349
|
+
: 'object';
|
|
311
350
|
// Type identifier as stored on `system.type`: a typename DXN for static schemas,
|
|
312
|
-
// an `echo:` EID for stored (dynamic) schemas.
|
|
313
|
-
|
|
351
|
+
// an `echo:` EID for stored (dynamic) schemas. Normalize the EID form so the indexed
|
|
352
|
+
// value matches the normalized value the query path compares against (legacy
|
|
353
|
+
// single-slash `echo:/<id>` and canonical `echo:///<id>` address the same schema).
|
|
354
|
+
// A preserved prior row was normalized when it was written, so it needs no re-normalizing.
|
|
355
|
+
const typeDXN = preserveBody
|
|
356
|
+
? priorRow.typeDXN
|
|
357
|
+
: URI.make(_normalizeTypeUri(castData[ATTR_TYPE] ? String(castData[ATTR_TYPE]) : 'type'));
|
|
314
358
|
const deleted = castData[ATTR_DELETED] ? 1 : 0;
|
|
315
359
|
// Relations.
|
|
316
|
-
const source =
|
|
317
|
-
|
|
360
|
+
const source = preserveBody
|
|
361
|
+
? priorRow.source
|
|
362
|
+
: entityKind === 'relation'
|
|
363
|
+
? (castData[ATTR_RELATION_SOURCE] ?? null)
|
|
364
|
+
: null;
|
|
365
|
+
const target = preserveBody
|
|
366
|
+
? priorRow.target
|
|
367
|
+
: entityKind === 'relation'
|
|
368
|
+
? (castData[ATTR_RELATION_TARGET] ?? null)
|
|
369
|
+
: null;
|
|
318
370
|
// Parent (nullable).
|
|
319
|
-
const parent = castData[ATTR_PARENT] ?? null;
|
|
371
|
+
const parent = preserveBody ? priorRow.parent : (castData[ATTR_PARENT] ?? null);
|
|
320
372
|
|
|
321
373
|
const updatedAtTimestamp = object.updatedAt;
|
|
322
374
|
// Prefer the creation timestamp stored in the document (survives compaction/migrations).
|
|
@@ -324,6 +376,10 @@ export class EntityMetaIndex implements Index {
|
|
|
324
376
|
const createdAtTimestamp = object.createdAt ?? updatedAtTimestamp;
|
|
325
377
|
|
|
326
378
|
if (existing.length > 0) {
|
|
379
|
+
// Feed entries collapse by id to the latest whole-object block — a re-append reusing
|
|
380
|
+
// an existing id (a live feed object's `Obj.update`) UPDATEs this row wholesale.
|
|
381
|
+
// TODO(wittjosiah): With partial-update blocks (see `EchoFeedCodec.encode`'s TODO),
|
|
382
|
+
// this becomes a field-level last-write-wins merge instead of a wholesale replace.
|
|
327
383
|
yield* sql`
|
|
328
384
|
UPDATE objectMeta SET
|
|
329
385
|
version = ${version},
|
|
@@ -519,7 +575,7 @@ export class EntityMetaIndex implements Index {
|
|
|
519
575
|
* Matches both:
|
|
520
576
|
* - Objects whose `parent` field references one of the given parent ids (standard parent/child hierarchy).
|
|
521
577
|
* - Queue items whose `queueId` equals one of the parent ids (e.g. items inside a Feed, since a feed's queue
|
|
522
|
-
* DXN uses the feed's object id as its queue id — see `Feed.
|
|
578
|
+
* DXN uses the feed's object id as its queue id — see `Feed.getFeedUri`).
|
|
523
579
|
*/
|
|
524
580
|
queryChildren = Effect.fn('EntityMetaIndex.queryChildren')(
|
|
525
581
|
(query: {
|
package/src/indexes/fts-index.ts
CHANGED
|
@@ -8,6 +8,7 @@ import type * as Statement from '@effect/sql/Statement';
|
|
|
8
8
|
import * as Effect from 'effect/Effect';
|
|
9
9
|
|
|
10
10
|
import type { Obj } from '@dxos/echo';
|
|
11
|
+
import { ATTR_TYPE } from '@dxos/echo/internal';
|
|
11
12
|
import type { EntityId, SpaceId } from '@dxos/keys';
|
|
12
13
|
|
|
13
14
|
import type { EntityMeta } from './entity-meta-index';
|
|
@@ -232,10 +233,27 @@ export class FtsIndex implements Index {
|
|
|
232
233
|
return yield* Effect.die(new Error('FtsIndex.update requires recordId to be set'));
|
|
233
234
|
}
|
|
234
235
|
|
|
235
|
-
const snapshot = JSON.stringify(data);
|
|
236
|
-
|
|
237
236
|
// FTS5 doesn't support UPDATE, need DELETE + INSERT for upsert.
|
|
238
|
-
const existing = yield* sql<{
|
|
237
|
+
const existing = yield* sql<{
|
|
238
|
+
rowid: number;
|
|
239
|
+
snapshot: string;
|
|
240
|
+
}>`SELECT rowid, snapshot FROM ftsIndex WHERE rowid = ${recordId}`;
|
|
241
|
+
|
|
242
|
+
// A partial block carries no `@type`/body — notably the `{ id, '@deleted': true }`
|
|
243
|
+
// tombstone appended by `Feed.remove`. Feed blocks are stored wholesale, so merge the
|
|
244
|
+
// partial onto the prior snapshot to retain the body and type while layering the new
|
|
245
|
+
// marker; otherwise the DELETE+INSERT below would replace the full snapshot with the
|
|
246
|
+
// bare partial and the client could not hydrate the deleted object (`Obj.fromJSON`
|
|
247
|
+
// needs `@type` to decode). Full blocks (carrying `@type`) still replace wholesale.
|
|
248
|
+
// TODO(wittjosiah): Generalise to field-level LWW once partial-update blocks exist
|
|
249
|
+
// (see `EchoFeedCodec.encode` and `EntityMetaIndex.update`).
|
|
250
|
+
const isPartialBlock = (data as Record<string, unknown>)[ATTR_TYPE] === undefined;
|
|
251
|
+
const merged =
|
|
252
|
+
isPartialBlock && existing.length > 0
|
|
253
|
+
? { ...(JSON.parse(existing[0].snapshot) as Record<string, unknown>), ...data }
|
|
254
|
+
: data;
|
|
255
|
+
const snapshot = JSON.stringify(merged);
|
|
256
|
+
|
|
239
257
|
if (existing.length > 0) {
|
|
240
258
|
yield* sql`DELETE FROM ftsIndex WHERE rowid = ${recordId}`;
|
|
241
259
|
}
|