@dxos/index-core 0.10.0 → 0.11.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@dxos/index-core",
3
- "version": "0.10.0",
3
+ "version": "0.11.0",
4
4
  "description": "Indexing core.",
5
5
  "homepage": "https://dxos.org",
6
6
  "bugs": "https://github.com/dxos/dxos/issues",
@@ -16,7 +16,7 @@
16
16
  ".": {
17
17
  "source": "./src/index.ts",
18
18
  "types": "./dist/types/src/index.d.ts",
19
- "default": "./dist/lib/neutral/index.mjs"
19
+ "import": "./dist/lib/index.mjs"
20
20
  }
21
21
  },
22
22
  "files": [
@@ -28,12 +28,12 @@
28
28
  "@effect/platform": "0.96.2",
29
29
  "@effect/sql": "0.51.1",
30
30
  "effect": "3.21.4",
31
- "@dxos/echo": "0.10.0",
32
- "@dxos/context": "0.10.0",
33
- "@dxos/echo-protocol": "0.10.0",
34
- "@dxos/invariant": "0.10.0",
35
- "@dxos/keys": "0.10.0",
36
- "@dxos/sql-sqlite": "0.10.0"
31
+ "@dxos/context": "0.11.0",
32
+ "@dxos/echo": "0.11.0",
33
+ "@dxos/echo-protocol": "0.11.0",
34
+ "@dxos/keys": "0.11.0",
35
+ "@dxos/invariant": "0.11.0",
36
+ "@dxos/sql-sqlite": "0.11.0"
37
37
  },
38
38
  "devDependencies": {
39
39
  "@effect/sql-sqlite-node": "0.52.0"
@@ -4,6 +4,7 @@
4
4
 
5
5
  import * as Reactivity from '@effect/experimental/Reactivity';
6
6
  import * as SqliteClient from '@effect/sql-sqlite-node/SqliteClient';
7
+ import * as SqlClient from '@effect/sql/SqlClient';
7
8
  import { describe, expect, it } from '@effect/vitest';
8
9
  import * as Effect from 'effect/Effect';
9
10
  import * as Layer from 'effect/Layer';
@@ -66,6 +67,46 @@ describe('EntityMetaIndex', () => {
66
67
  }).pipe(Effect.provide(TestLayer)),
67
68
  );
68
69
 
70
+ it.effect('resolves a legacy single-slash type-identifier row when queried by the canonical form', () =>
71
+ Effect.gen(function* () {
72
+ const index = new EntityMetaIndex();
73
+ yield* index.migrate();
74
+ const sql = yield* SqlClient.SqlClient;
75
+
76
+ const spaceId = SpaceId.random();
77
+ const objectId = EntityId.random();
78
+ const schemaId = EntityId.random();
79
+
80
+ const item: IndexerObject = {
81
+ spaceId,
82
+ queueId: EntityId.random(),
83
+ queueNamespace: 'data',
84
+ documentId: null,
85
+ recordId: null,
86
+ createdAt: null,
87
+ updatedAt: Date.now(),
88
+ data: {
89
+ id: objectId,
90
+ [ATTR_TYPE]: EID.make({ entityId: schemaId }),
91
+ [ATTR_DELETED]: false,
92
+ },
93
+ };
94
+
95
+ yield* index.update([item]);
96
+
97
+ // The write path normalizes identifiers, so rewrite the row to the legacy single-slash form to
98
+ // simulate a pre-normalization on-disk row (parameterized to avoid SQL injection).
99
+ const legacyTypeDxn = `echo:/${schemaId}`;
100
+ yield* sql`UPDATE objectMeta SET typeDXN = ${legacyTypeDxn} WHERE objectId = ${objectId}`;
101
+ const [stored] = yield* sql<{ typeDXN: string }>`SELECT typeDXN FROM objectMeta WHERE objectId = ${objectId}`;
102
+ expect(stored.typeDXN).toBe(legacyTypeDxn);
103
+
104
+ // A canonical-form query must still resolve the legacy row without a reindex.
105
+ const results = yield* index.query({ spaceId, typeDXN: EID.make({ entityId: schemaId }) });
106
+ expect(results.map((meta) => meta.objectId)).toEqual([objectId]);
107
+ }).pipe(Effect.provide(TestLayer)),
108
+ );
109
+
69
110
  it.effect('should not treat LIKE wildcards in versionless type queries', () =>
70
111
  Effect.gen(function* () {
71
112
  const index = new EntityMetaIndex();
@@ -16,8 +16,8 @@ import type { Index } from './interface';
16
16
 
17
17
  /**
18
18
  * Normalizes an echo: EID to the local (unqualified) form so SQL comparisons are consistent.
19
- * DB rows always store `echo:/<entityId>`; a space-qualified `echo://<space>/<entityId>` would
20
- * otherwise miss every row for that type.
19
+ * Rows are indexed in the canonical `echo:///<entityId>` form; a space-qualified
20
+ * `echo://<space>/<entityId>` would otherwise miss every row for that type.
21
21
  */
22
22
  const _normalizeTypeUri = (typeDXN: string): string => {
23
23
  if (!typeDXN.startsWith('echo:')) {
@@ -31,6 +31,18 @@ const _normalizeTypeUri = (typeDXN: string): string => {
31
31
  return entityId ? EID.make({ entityId }) : typeDXN;
32
32
  };
33
33
 
34
+ /**
35
+ * Every stored `typeDXN` form equivalent to a normalized type identifier. Rows written before
36
+ * `echo:///<id>` became the canonical local EID hold the single-slash `echo:/<id>` form; both
37
+ * address the same stored schema, so queries must match either without requiring a reindex.
38
+ */
39
+ const _typeUriEquivalents = (normalized: string): string[] => {
40
+ if (normalized.startsWith('echo:///')) {
41
+ return [normalized, `echo:/${normalized.slice('echo:///'.length)}`];
42
+ }
43
+ return [normalized];
44
+ };
45
+
34
46
  const _escapeLikePrefix = (prefix: string) => {
35
47
  // Escape LIKE metacharacters in the *literal* prefix (we still append a wildcard for the version suffix).
36
48
  // Backslash is used as the ESCAPE character.
@@ -150,13 +162,16 @@ export class EntityMetaIndex implements Index {
150
162
  const normalizedTypeDXN = _normalizeTypeUri(query.typeDXN);
151
163
  const parsedDxn = DXN.isDXN(normalizedTypeDXN) ? normalizedTypeDXN : undefined;
152
164
  const hasNoVersion = parsedDxn !== undefined && DXN.getVersion(parsedDxn) === undefined;
165
+ const forms = _typeUriEquivalents(normalizedTypeDXN);
166
+ const exactMatch = sql.or(forms.map((form) => sql`typeDXN = ${form}`));
167
+ // Version wildcard must cover every equivalent form so a versionless query still matches
168
+ // legacy versioned rows written under the single-slash prefix.
169
+ const likeMatch = sql.or(forms.map((form) => sql`typeDXN LIKE ${_escapeLikePrefix(form)} ESCAPE '\\'`));
153
170
 
154
171
  // SQLite stores booleans as integers, so we need to specify the raw row type.
155
172
  const rows = hasNoVersion
156
- ? yield* sql<EntityMeta>`SELECT * FROM objectMeta WHERE spaceId = ${query.spaceId} AND (typeDXN = ${
157
- normalizedTypeDXN
158
- } OR typeDXN LIKE ${_escapeLikePrefix(normalizedTypeDXN)} ESCAPE '\\')`
159
- : yield* sql<EntityMeta>`SELECT * FROM objectMeta WHERE spaceId = ${query.spaceId} AND typeDXN = ${normalizedTypeDXN}`;
173
+ ? yield* sql<EntityMeta>`SELECT * FROM objectMeta WHERE spaceId = ${query.spaceId} AND (${exactMatch} OR ${likeMatch})`
174
+ : yield* sql<EntityMeta>`SELECT * FROM objectMeta WHERE spaceId = ${query.spaceId} AND ${exactMatch}`;
160
175
  return rows.map((row) => ({
161
176
  ...row,
162
177
  deleted: !!row.deleted,
@@ -229,9 +244,13 @@ export class EntityMetaIndex implements Index {
229
244
  const normalized = _normalizeTypeUri(typeDXN);
230
245
  const parsedDxn = DXN.isDXN(normalized) ? normalized : undefined;
231
246
  const hasNoVersion = parsedDxn !== undefined && DXN.getVersion(parsedDxn) === undefined;
232
- const exactMatch = sql`typeDXN = ${normalized}`;
247
+ const forms = _typeUriEquivalents(normalized);
248
+ const exactMatch = sql.or(forms.map((form) => sql`typeDXN = ${form}`));
233
249
  return hasNoVersion
234
- ? sql.or([exactMatch, sql`typeDXN LIKE ${_escapeLikePrefix(normalized)} ESCAPE '\\'`])
250
+ ? sql.or([
251
+ exactMatch,
252
+ sql.or(forms.map((form) => sql`typeDXN LIKE ${_escapeLikePrefix(form)} ESCAPE '\\'`)),
253
+ ])
235
254
  : exactMatch;
236
255
  }),
237
256
  );
@@ -287,15 +306,21 @@ export class EntityMetaIndex implements Index {
287
306
  const objectId = castData.id;
288
307
 
289
308
  // Check for existing record by (spaceId, queueId) or (spaceId, documentId).
290
- let existing: readonly { recordId: number }[];
309
+ type ExistingRow = {
310
+ recordId: number;
311
+ entityKind: string;
312
+ typeDXN: string;
313
+ source: string | null;
314
+ target: string | null;
315
+ parent: string | null;
316
+ };
317
+ let existing: readonly ExistingRow[];
291
318
  if (documentId) {
292
- existing = yield* sql<{
293
- recordId: number;
294
- }>`SELECT recordId FROM objectMeta WHERE spaceId = ${spaceId} AND documentId = ${documentId} AND objectId = ${objectId} LIMIT 1`;
319
+ existing =
320
+ yield* sql<ExistingRow>`SELECT recordId, entityKind, typeDXN, source, target, parent FROM objectMeta WHERE spaceId = ${spaceId} AND documentId = ${documentId} AND objectId = ${objectId} LIMIT 1`;
295
321
  } else if (queueId) {
296
- existing = yield* sql<{
297
- recordId: number;
298
- }>`SELECT recordId FROM objectMeta WHERE spaceId = ${spaceId} AND queueId = ${queueId} AND objectId = ${objectId} LIMIT 1`;
322
+ existing =
323
+ yield* sql<ExistingRow>`SELECT recordId, entityKind, typeDXN, source, target, parent FROM objectMeta WHERE spaceId = ${spaceId} AND queueId = ${queueId} AND objectId = ${objectId} LIMIT 1`;
299
324
  } else {
300
325
  // Should not happen based on IndexerObject definition (one must be present ideally), but handle gracefully.
301
326
  existing = [];
@@ -306,17 +331,44 @@ export class EntityMetaIndex implements Index {
306
331
  const [{ v }] = result;
307
332
  const version = (v ?? 0) + 1;
308
333
 
334
+ // A partial block carries no `@type`/body — notably the `{ id, '@deleted': true }`
335
+ // tombstone appended by `Feed.remove`. Preserve the prior row's body-derived columns
336
+ // (type, kind, relation endpoints, parent) rather than recomputing them from the empty
337
+ // block; otherwise `typeDXN` collapses to the `'type'` fallback and type-scoped queries
338
+ // with `deleted: 'include'` stop matching the deleted object. Only `deleted`/`version`/
339
+ // `updatedAt` advance for such a block.
340
+ const priorRow = existing.length > 0 ? existing[0] : undefined;
341
+ const isPartialBlock = castData[ATTR_TYPE] === undefined;
342
+ const preserveBody = isPartialBlock && priorRow !== undefined;
343
+
309
344
  // Extract metadata.
310
- const entityKind = castData[ATTR_RELATION_SOURCE] ? 'relation' : 'object';
345
+ const entityKind = preserveBody
346
+ ? priorRow.entityKind
347
+ : castData[ATTR_RELATION_SOURCE]
348
+ ? 'relation'
349
+ : 'object';
311
350
  // Type identifier as stored on `system.type`: a typename DXN for static schemas,
312
- // an `echo:` EID for stored (dynamic) schemas.
313
- const typeDXN = URI.make(castData[ATTR_TYPE] ? String(castData[ATTR_TYPE]) : 'type');
351
+ // an `echo:` EID for stored (dynamic) schemas. Normalize the EID form so the indexed
352
+ // value matches the normalized value the query path compares against (legacy
353
+ // single-slash `echo:/<id>` and canonical `echo:///<id>` address the same schema).
354
+ // A preserved prior row was normalized when it was written, so it needs no re-normalizing.
355
+ const typeDXN = preserveBody
356
+ ? priorRow.typeDXN
357
+ : URI.make(_normalizeTypeUri(castData[ATTR_TYPE] ? String(castData[ATTR_TYPE]) : 'type'));
314
358
  const deleted = castData[ATTR_DELETED] ? 1 : 0;
315
359
  // Relations.
316
- const source = entityKind === 'relation' ? (castData[ATTR_RELATION_SOURCE] ?? null) : null;
317
- const target = entityKind === 'relation' ? (castData[ATTR_RELATION_TARGET] ?? null) : null;
360
+ const source = preserveBody
361
+ ? priorRow.source
362
+ : entityKind === 'relation'
363
+ ? (castData[ATTR_RELATION_SOURCE] ?? null)
364
+ : null;
365
+ const target = preserveBody
366
+ ? priorRow.target
367
+ : entityKind === 'relation'
368
+ ? (castData[ATTR_RELATION_TARGET] ?? null)
369
+ : null;
318
370
  // Parent (nullable).
319
- const parent = castData[ATTR_PARENT] ?? null;
371
+ const parent = preserveBody ? priorRow.parent : (castData[ATTR_PARENT] ?? null);
320
372
 
321
373
  const updatedAtTimestamp = object.updatedAt;
322
374
  // Prefer the creation timestamp stored in the document (survives compaction/migrations).
@@ -324,6 +376,10 @@ export class EntityMetaIndex implements Index {
324
376
  const createdAtTimestamp = object.createdAt ?? updatedAtTimestamp;
325
377
 
326
378
  if (existing.length > 0) {
379
+ // Feed entries collapse by id to the latest whole-object block — a re-append reusing
380
+ // an existing id (a live feed object's `Obj.update`) UPDATEs this row wholesale.
381
+ // TODO(wittjosiah): With partial-update blocks (see `EchoFeedCodec.encode`'s TODO),
382
+ // this becomes a field-level last-write-wins merge instead of a wholesale replace.
327
383
  yield* sql`
328
384
  UPDATE objectMeta SET
329
385
  version = ${version},
@@ -519,7 +575,7 @@ export class EntityMetaIndex implements Index {
519
575
  * Matches both:
520
576
  * - Objects whose `parent` field references one of the given parent ids (standard parent/child hierarchy).
521
577
  * - Queue items whose `queueId` equals one of the parent ids (e.g. items inside a Feed, since a feed's queue
522
- * DXN uses the feed's object id as its queue id — see `Feed.getQueueUri`).
578
+ * DXN uses the feed's object id as its queue id — see `Feed.getFeedUri`).
523
579
  */
524
580
  queryChildren = Effect.fn('EntityMetaIndex.queryChildren')(
525
581
  (query: {
@@ -8,6 +8,7 @@ import type * as Statement from '@effect/sql/Statement';
8
8
  import * as Effect from 'effect/Effect';
9
9
 
10
10
  import type { Obj } from '@dxos/echo';
11
+ import { ATTR_TYPE } from '@dxos/echo/internal';
11
12
  import type { EntityId, SpaceId } from '@dxos/keys';
12
13
 
13
14
  import type { EntityMeta } from './entity-meta-index';
@@ -232,10 +233,27 @@ export class FtsIndex implements Index {
232
233
  return yield* Effect.die(new Error('FtsIndex.update requires recordId to be set'));
233
234
  }
234
235
 
235
- const snapshot = JSON.stringify(data);
236
-
237
236
  // FTS5 doesn't support UPDATE, need DELETE + INSERT for upsert.
238
- const existing = yield* sql<{ rowid: number }>`SELECT rowid FROM ftsIndex WHERE rowid = ${recordId}`;
237
+ const existing = yield* sql<{
238
+ rowid: number;
239
+ snapshot: string;
240
+ }>`SELECT rowid, snapshot FROM ftsIndex WHERE rowid = ${recordId}`;
241
+
242
+ // A partial block carries no `@type`/body — notably the `{ id, '@deleted': true }`
243
+ // tombstone appended by `Feed.remove`. Feed blocks are stored wholesale, so merge the
244
+ // partial onto the prior snapshot to retain the body and type while layering the new
245
+ // marker; otherwise the DELETE+INSERT below would replace the full snapshot with the
246
+ // bare partial and the client could not hydrate the deleted object (`Obj.fromJSON`
247
+ // needs `@type` to decode). Full blocks (carrying `@type`) still replace wholesale.
248
+ // TODO(wittjosiah): Generalise to field-level LWW once partial-update blocks exist
249
+ // (see `EchoFeedCodec.encode` and `EntityMetaIndex.update`).
250
+ const isPartialBlock = (data as Record<string, unknown>)[ATTR_TYPE] === undefined;
251
+ const merged =
252
+ isPartialBlock && existing.length > 0
253
+ ? { ...(JSON.parse(existing[0].snapshot) as Record<string, unknown>), ...data }
254
+ : data;
255
+ const snapshot = JSON.stringify(merged);
256
+
239
257
  if (existing.length > 0) {
240
258
  yield* sql`DELETE FROM ftsIndex WHERE rowid = ${recordId}`;
241
259
  }