@dxos/index-core 0.8.4-main.c85a9c8dae → 0.8.4-main.e00bdcdb52
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/lib/neutral/index.mjs +73 -22
- package/dist/lib/neutral/index.mjs.map +3 -3
- package/dist/lib/neutral/meta.json +1 -1
- package/dist/types/src/index-engine.d.ts +13 -3
- package/dist/types/src/index-engine.d.ts.map +1 -1
- package/dist/types/src/index-tracker.d.ts +3 -3
- package/dist/types/src/index-tracker.d.ts.map +1 -1
- package/dist/types/src/indexes/fts-index.d.ts +1 -1
- package/dist/types/src/indexes/fts-index.d.ts.map +1 -1
- package/dist/types/src/indexes/interface.d.ts +4 -0
- package/dist/types/src/indexes/interface.d.ts.map +1 -1
- package/dist/types/src/indexes/object-meta-index.d.ts +24 -4
- package/dist/types/src/indexes/object-meta-index.d.ts.map +1 -1
- package/dist/types/src/indexes/reverse-ref-index.d.ts.map +1 -1
- package/dist/types/tsconfig.tsbuildinfo +1 -1
- package/package.json +12 -12
- package/src/index-engine.test.ts +50 -3
- package/src/index-engine.ts +27 -5
- package/src/index-tracker.ts +9 -0
- package/src/indexes/fts-index.test.ts +21 -3
- package/src/indexes/interface.ts +5 -0
- package/src/indexes/object-meta-index.test.ts +114 -8
- package/src/indexes/object-meta-index.ts +78 -5
- package/src/indexes/reverse-ref-index.test.ts +9 -2
- package/src/indexes/reverse-ref-index.ts +0 -1
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@dxos/index-core",
|
|
3
|
-
"version": "0.8.4-main.
|
|
3
|
+
"version": "0.8.4-main.e00bdcdb52",
|
|
4
4
|
"description": "Indexing core.",
|
|
5
5
|
"homepage": "https://dxos.org",
|
|
6
6
|
"bugs": "https://github.com/dxos/dxos/issues",
|
|
@@ -28,17 +28,17 @@
|
|
|
28
28
|
"@effect/experimental": "0.58.0",
|
|
29
29
|
"@effect/platform": "0.94.4",
|
|
30
30
|
"@effect/sql": "0.49.0",
|
|
31
|
-
"effect": "3.
|
|
32
|
-
"@dxos/async": "0.8.4-main.
|
|
33
|
-
"@dxos/
|
|
34
|
-
"@dxos/
|
|
35
|
-
"@dxos/echo-protocol": "0.8.4-main.
|
|
36
|
-
"@dxos/
|
|
37
|
-
"@dxos/effect": "0.8.4-main.
|
|
38
|
-
"@dxos/
|
|
39
|
-
"@dxos/
|
|
40
|
-
"@dxos/log": "0.8.4-main.
|
|
41
|
-
"@dxos/sql-sqlite": "0.8.4-main.
|
|
31
|
+
"effect": "3.20.0",
|
|
32
|
+
"@dxos/async": "0.8.4-main.e00bdcdb52",
|
|
33
|
+
"@dxos/context": "0.8.4-main.e00bdcdb52",
|
|
34
|
+
"@dxos/debug": "0.8.4-main.e00bdcdb52",
|
|
35
|
+
"@dxos/echo-protocol": "0.8.4-main.e00bdcdb52",
|
|
36
|
+
"@dxos/echo": "0.8.4-main.e00bdcdb52",
|
|
37
|
+
"@dxos/effect": "0.8.4-main.e00bdcdb52",
|
|
38
|
+
"@dxos/invariant": "0.8.4-main.e00bdcdb52",
|
|
39
|
+
"@dxos/keys": "0.8.4-main.e00bdcdb52",
|
|
40
|
+
"@dxos/log": "0.8.4-main.e00bdcdb52",
|
|
41
|
+
"@dxos/sql-sqlite": "0.8.4-main.e00bdcdb52"
|
|
42
42
|
},
|
|
43
43
|
"devDependencies": {
|
|
44
44
|
"@effect/sql-sqlite-node": "0.50.1"
|
package/src/index-engine.test.ts
CHANGED
|
@@ -8,6 +8,7 @@ import { describe, expect, it } from '@effect/vitest';
|
|
|
8
8
|
import * as Effect from 'effect/Effect';
|
|
9
9
|
import * as Layer from 'effect/Layer';
|
|
10
10
|
|
|
11
|
+
import { Context } from '@dxos/context';
|
|
11
12
|
import { ATTR_TYPE } from '@dxos/echo/internal';
|
|
12
13
|
import { invariant } from '@dxos/invariant';
|
|
13
14
|
import { DXN, ObjectId, SpaceId } from '@dxos/keys';
|
|
@@ -49,6 +50,7 @@ class MockIndexDataSource implements IndexDataSource {
|
|
|
49
50
|
}
|
|
50
51
|
|
|
51
52
|
getChangedObjects(
|
|
53
|
+
_ctx: Context,
|
|
52
54
|
cursors: IndexCursor[],
|
|
53
55
|
opts?: { limit?: number },
|
|
54
56
|
): Effect.Effect<{ objects: IndexerObject[]; cursors: DataSourceCursor[] }> {
|
|
@@ -118,6 +120,7 @@ describe('IndexEngine', () => {
|
|
|
118
120
|
documentId: 'doc-1',
|
|
119
121
|
queueId: null,
|
|
120
122
|
recordId: null,
|
|
123
|
+
updatedAt: Date.now(),
|
|
121
124
|
data: {
|
|
122
125
|
id: ObjectId.random(),
|
|
123
126
|
[ATTR_TYPE]: TYPE_DEFAULT,
|
|
@@ -128,7 +131,7 @@ describe('IndexEngine', () => {
|
|
|
128
131
|
dataSource.push([obj1]);
|
|
129
132
|
|
|
130
133
|
// First update.
|
|
131
|
-
const { updated } = yield* engine.update(dataSource, { spaceId: null });
|
|
134
|
+
const { updated } = yield* engine.update(Context.default(), dataSource, { spaceId: null });
|
|
132
135
|
// Updates objectMeta, FTS, and reverseRef indexes.
|
|
133
136
|
expect(updated).toBe(2);
|
|
134
137
|
|
|
@@ -154,12 +157,13 @@ describe('IndexEngine', () => {
|
|
|
154
157
|
documentId: obj1.documentId,
|
|
155
158
|
queueId: null,
|
|
156
159
|
recordId: null,
|
|
160
|
+
updatedAt: Date.now(),
|
|
157
161
|
data: { id: obj1.data.id, [ATTR_TYPE]: obj1.data[ATTR_TYPE], title: 'Hello World' },
|
|
158
162
|
};
|
|
159
163
|
dataSource.push([obj1Updated]);
|
|
160
164
|
|
|
161
165
|
// Second update.
|
|
162
|
-
const { updated: updated2 } = yield* engine.update(dataSource, { spaceId: null });
|
|
166
|
+
const { updated: updated2 } = yield* engine.update(Context.default(), dataSource, { spaceId: null });
|
|
163
167
|
expect(updated2).toBe(2);
|
|
164
168
|
|
|
165
169
|
// Verify update.
|
|
@@ -193,6 +197,7 @@ describe('IndexEngine', () => {
|
|
|
193
197
|
queueId: null,
|
|
194
198
|
documentId: 'd1',
|
|
195
199
|
recordId: null,
|
|
200
|
+
updatedAt: Date.now(),
|
|
196
201
|
data: {
|
|
197
202
|
id: ObjectId.random(),
|
|
198
203
|
[ATTR_TYPE]: TYPE_A,
|
|
@@ -204,6 +209,7 @@ describe('IndexEngine', () => {
|
|
|
204
209
|
queueId: null,
|
|
205
210
|
documentId: 'd2',
|
|
206
211
|
recordId: null,
|
|
212
|
+
updatedAt: Date.now(),
|
|
207
213
|
data: {
|
|
208
214
|
id: ObjectId.random(),
|
|
209
215
|
[ATTR_TYPE]: TYPE_A,
|
|
@@ -215,6 +221,7 @@ describe('IndexEngine', () => {
|
|
|
215
221
|
queueId: null,
|
|
216
222
|
documentId: 'd3',
|
|
217
223
|
recordId: null,
|
|
224
|
+
updatedAt: Date.now(),
|
|
218
225
|
data: {
|
|
219
226
|
id: ObjectId.random(),
|
|
220
227
|
[ATTR_TYPE]: TYPE_B,
|
|
@@ -225,7 +232,7 @@ describe('IndexEngine', () => {
|
|
|
225
232
|
|
|
226
233
|
dataSource.push(objects);
|
|
227
234
|
|
|
228
|
-
yield* engine.update(dataSource, { spaceId: null });
|
|
235
|
+
yield* engine.update(Context.default(), dataSource, { spaceId: null });
|
|
229
236
|
|
|
230
237
|
const resultsA = yield* metaIndex.query({ spaceId: spaceId.toString(), typeDxn: TYPE_A });
|
|
231
238
|
expect(resultsA).toHaveLength(2);
|
|
@@ -242,4 +249,44 @@ describe('IndexEngine', () => {
|
|
|
242
249
|
expect(ftsResults).toHaveLength(2);
|
|
243
250
|
}, Effect.provide(TestLayer)),
|
|
244
251
|
);
|
|
252
|
+
|
|
253
|
+
it.effect(
|
|
254
|
+
'done is true only when all sub-indexes have no remaining work',
|
|
255
|
+
Effect.fnUntraced(function* () {
|
|
256
|
+
const { tracker, metaIndex, ftsIndex, reverseRefIndex } = yield* setup;
|
|
257
|
+
|
|
258
|
+
const engine = new IndexEngine({ tracker, objectMetaIndex: metaIndex, ftsIndex, reverseRefIndex });
|
|
259
|
+
const dataSource = new MockIndexDataSource();
|
|
260
|
+
const spaceId = SpaceId.random();
|
|
261
|
+
|
|
262
|
+
// First update with no data — both sub-indexes report done immediately.
|
|
263
|
+
const { updated: updated0, done: done0 } = yield* engine.update(Context.default(), dataSource, { spaceId: null });
|
|
264
|
+
expect(updated0).toBe(0);
|
|
265
|
+
expect(done0).toBe(true);
|
|
266
|
+
|
|
267
|
+
// Push an object so sub-indexes have work to do.
|
|
268
|
+
dataSource.push([
|
|
269
|
+
{
|
|
270
|
+
spaceId,
|
|
271
|
+
queueId: null,
|
|
272
|
+
documentId: 'doc-done-test',
|
|
273
|
+
recordId: null,
|
|
274
|
+
updatedAt: Date.now(),
|
|
275
|
+
data: { id: ObjectId.random(), [ATTR_TYPE]: TYPE_DEFAULT, title: 'Done test' },
|
|
276
|
+
},
|
|
277
|
+
]);
|
|
278
|
+
|
|
279
|
+
// Update with pending data — sub-indexes process objects, done is false.
|
|
280
|
+
const { updated: updated1, done: done1 } = yield* engine.update(Context.default(), dataSource, { spaceId: null });
|
|
281
|
+
expect(updated1).toBeGreaterThan(0);
|
|
282
|
+
expect(done1).toBe(false);
|
|
283
|
+
|
|
284
|
+
// Second update with no new data — all sub-indexes caught up, done is true.
|
|
285
|
+
const { updated: updated2, done: done2 } = yield* engine.update(Context.default(), dataSource, {
|
|
286
|
+
spaceId: null,
|
|
287
|
+
});
|
|
288
|
+
expect(updated2).toBe(0);
|
|
289
|
+
expect(done2).toBe(true);
|
|
290
|
+
}, Effect.provide(TestLayer)),
|
|
291
|
+
);
|
|
245
292
|
});
|
package/src/index-engine.ts
CHANGED
|
@@ -6,6 +6,7 @@ import type * as SqlClient from '@effect/sql/SqlClient';
|
|
|
6
6
|
import type * as SqlError from '@effect/sql/SqlError';
|
|
7
7
|
import * as Effect from 'effect/Effect';
|
|
8
8
|
|
|
9
|
+
import { type Context } from '@dxos/context';
|
|
9
10
|
import type { ObjectId, SpaceId } from '@dxos/keys';
|
|
10
11
|
import * as SqlTransaction from '@dxos/sql-sqlite/SqlTransaction';
|
|
11
12
|
|
|
@@ -29,7 +30,7 @@ export interface DataSourceCursor {
|
|
|
29
30
|
spaceId: SpaceId | null;
|
|
30
31
|
|
|
31
32
|
/**
|
|
32
|
-
* documentId or
|
|
33
|
+
* documentId or queueNamespace.
|
|
33
34
|
*/
|
|
34
35
|
resourceId: string | null;
|
|
35
36
|
|
|
@@ -43,6 +44,7 @@ export interface IndexDataSource {
|
|
|
43
44
|
readonly sourceName: string; // e.g. queue, automerge, etc.
|
|
44
45
|
|
|
45
46
|
getChangedObjects(
|
|
47
|
+
ctx: Context,
|
|
46
48
|
cursors: DataSourceCursor[],
|
|
47
49
|
opts?: { limit?: number },
|
|
48
50
|
): Effect.Effect<{ objects: IndexerObject[]; cursors: DataSourceCursor[] }>;
|
|
@@ -132,6 +134,18 @@ export class IndexEngine {
|
|
|
132
134
|
}): Effect.Effect<readonly ObjectMeta[], SqlError.SqlError, SqlClient.SqlClient> {
|
|
133
135
|
return this.#objectMetaIndex.queryTypes(query);
|
|
134
136
|
}
|
|
137
|
+
queryByTimeRange(query: {
|
|
138
|
+
spaceIds: readonly string[];
|
|
139
|
+
updatedAfter?: number;
|
|
140
|
+
updatedBefore?: number;
|
|
141
|
+
createdAfter?: number;
|
|
142
|
+
createdBefore?: number;
|
|
143
|
+
includeAllQueues?: boolean;
|
|
144
|
+
queueIds?: readonly string[] | null;
|
|
145
|
+
}): Effect.Effect<readonly ObjectMeta[], SqlError.SqlError, SqlClient.SqlClient> {
|
|
146
|
+
return this.#objectMetaIndex.queryByTimeRange(query);
|
|
147
|
+
}
|
|
148
|
+
|
|
135
149
|
queryRelations(query: {
|
|
136
150
|
endpoint: 'source' | 'target';
|
|
137
151
|
anchorDxns: readonly string[];
|
|
@@ -151,6 +165,7 @@ export class IndexEngine {
|
|
|
151
165
|
}
|
|
152
166
|
|
|
153
167
|
update(
|
|
168
|
+
ctx: Context,
|
|
154
169
|
dataSource: IndexDataSource,
|
|
155
170
|
opts: { spaceId: SpaceId | null; limit?: number },
|
|
156
171
|
): Effect.Effect<
|
|
@@ -160,15 +175,18 @@ export class IndexEngine {
|
|
|
160
175
|
> {
|
|
161
176
|
return Effect.gen(this, function* () {
|
|
162
177
|
let updated = 0;
|
|
178
|
+
let done = true;
|
|
163
179
|
|
|
164
|
-
const { updated: updatedFtsIndex, done: doneFtsIndex } = yield* this.#update(this.#ftsIndex, dataSource, {
|
|
165
|
-
indexName: '
|
|
180
|
+
const { updated: updatedFtsIndex, done: doneFtsIndex } = yield* this.#update(ctx, this.#ftsIndex, dataSource, {
|
|
181
|
+
indexName: 'fts5',
|
|
166
182
|
spaceId: opts.spaceId,
|
|
167
183
|
limit: opts.limit,
|
|
168
184
|
});
|
|
169
185
|
updated += updatedFtsIndex;
|
|
186
|
+
done = done && doneFtsIndex;
|
|
170
187
|
|
|
171
188
|
const { updated: updatedReverseRefIndex, done: doneReverseRefIndex } = yield* this.#update(
|
|
189
|
+
ctx,
|
|
172
190
|
this.#reverseRefIndex,
|
|
173
191
|
dataSource,
|
|
174
192
|
{
|
|
@@ -178,8 +196,9 @@ export class IndexEngine {
|
|
|
178
196
|
},
|
|
179
197
|
);
|
|
180
198
|
updated += updatedReverseRefIndex;
|
|
199
|
+
done = done && doneReverseRefIndex;
|
|
181
200
|
|
|
182
|
-
return { updated, done
|
|
201
|
+
return { updated, done };
|
|
183
202
|
}).pipe(Effect.withSpan('IndexEngine.update'));
|
|
184
203
|
}
|
|
185
204
|
|
|
@@ -193,6 +212,7 @@ export class IndexEngine {
|
|
|
193
212
|
* 5. Updates the dependent index.
|
|
194
213
|
*/
|
|
195
214
|
#update(
|
|
215
|
+
ctx: Context,
|
|
196
216
|
index: Index,
|
|
197
217
|
source: IndexDataSource,
|
|
198
218
|
opts: { indexName: string; spaceId: SpaceId | null; limit?: number },
|
|
@@ -212,7 +232,9 @@ export class IndexEngine {
|
|
|
212
232
|
// Pass undefined to get all cursors when spaceId is null.
|
|
213
233
|
spaceId: opts.spaceId ?? undefined,
|
|
214
234
|
});
|
|
215
|
-
const { objects, cursors: updatedCursors } = yield* source.getChangedObjects(cursors, {
|
|
235
|
+
const { objects, cursors: updatedCursors } = yield* source.getChangedObjects(ctx, cursors, {
|
|
236
|
+
limit: opts.limit,
|
|
237
|
+
});
|
|
216
238
|
if (objects.length === 0) {
|
|
217
239
|
return { updated: 0, done: true };
|
|
218
240
|
}
|
package/src/index-tracker.ts
CHANGED
|
@@ -35,6 +35,11 @@ export const IndexCursor = Schema.Struct({
|
|
|
35
35
|
});
|
|
36
36
|
export interface IndexCursor extends Schema.Schema.Type<typeof IndexCursor> {}
|
|
37
37
|
|
|
38
|
+
/**
|
|
39
|
+
* Deprecated index names that are no longer used. Will be cleaned up on migration.
|
|
40
|
+
*/
|
|
41
|
+
const DEPRECATED_INDEX_NAMES = ['fts'];
|
|
42
|
+
|
|
38
43
|
export class IndexTracker {
|
|
39
44
|
migrate = Effect.fn('IndexTracker.migrate')(function* () {
|
|
40
45
|
const sql = yield* SqlClient.SqlClient;
|
|
@@ -49,6 +54,10 @@ export class IndexTracker {
|
|
|
49
54
|
cursor,
|
|
50
55
|
PRIMARY KEY (indexName, spaceId, sourceName, resourceId)
|
|
51
56
|
)`;
|
|
57
|
+
|
|
58
|
+
yield* Effect.forEach(DEPRECATED_INDEX_NAMES, (indexName) => {
|
|
59
|
+
return sql`DELETE FROM indexCursor WHERE indexName = ${indexName}`;
|
|
60
|
+
});
|
|
52
61
|
});
|
|
53
62
|
|
|
54
63
|
queryCursors = Effect.fn('IndexTracker.queryCursors')(
|
|
@@ -3,8 +3,8 @@
|
|
|
3
3
|
//
|
|
4
4
|
|
|
5
5
|
import * as Reactivity from '@effect/experimental/Reactivity';
|
|
6
|
-
import * as SqlClient from '@effect/sql/SqlClient';
|
|
7
6
|
import * as SqliteClient from '@effect/sql-sqlite-node/SqliteClient';
|
|
7
|
+
import * as SqlClient from '@effect/sql/SqlClient';
|
|
8
8
|
import { describe, expect, it } from '@effect/vitest';
|
|
9
9
|
import * as Effect from 'effect/Effect';
|
|
10
10
|
import * as Layer from 'effect/Layer';
|
|
@@ -16,7 +16,7 @@ import { FtsIndex } from './fts-index';
|
|
|
16
16
|
import type { IndexerObject } from './interface';
|
|
17
17
|
import { ObjectMetaIndex } from './object-meta-index';
|
|
18
18
|
|
|
19
|
-
const TYPE_PERSON = DXN.parse('dxn:type:example.
|
|
19
|
+
const TYPE_PERSON = DXN.parse('dxn:type:com.example.type.person:0.1.0').toString();
|
|
20
20
|
const TYPE_DEFAULT = DXN.parse('dxn:type:test.com/type/Type:0.1.0').toString();
|
|
21
21
|
|
|
22
22
|
const TestLayer = Layer.merge(
|
|
@@ -57,6 +57,7 @@ describe('FtsIndex', () => {
|
|
|
57
57
|
queueId: null,
|
|
58
58
|
documentId: 'doc-1',
|
|
59
59
|
recordId: null,
|
|
60
|
+
updatedAt: Date.now(),
|
|
60
61
|
data: {
|
|
61
62
|
id: ObjectId.random(),
|
|
62
63
|
[ATTR_TYPE]: TYPE_PERSON,
|
|
@@ -101,9 +102,10 @@ describe('FtsIndex', () => {
|
|
|
101
102
|
queueId: null,
|
|
102
103
|
documentId: 'doc-1',
|
|
103
104
|
recordId: null,
|
|
105
|
+
updatedAt: Date.now(),
|
|
104
106
|
data: {
|
|
105
107
|
id: objectId,
|
|
106
|
-
[ATTR_TYPE]: DXN.parse('dxn:type:example.
|
|
108
|
+
[ATTR_TYPE]: DXN.parse('dxn:type:com.example.type.person:0.1.0').toString(),
|
|
107
109
|
title: 'Original Title',
|
|
108
110
|
},
|
|
109
111
|
};
|
|
@@ -120,6 +122,7 @@ describe('FtsIndex', () => {
|
|
|
120
122
|
queueId: null,
|
|
121
123
|
documentId: 'doc-1',
|
|
122
124
|
recordId: null,
|
|
125
|
+
updatedAt: Date.now(),
|
|
123
126
|
data: {
|
|
124
127
|
id: objectId,
|
|
125
128
|
[ATTR_TYPE]: TYPE_DEFAULT,
|
|
@@ -157,6 +160,7 @@ describe('FtsIndex', () => {
|
|
|
157
160
|
queueId: null,
|
|
158
161
|
documentId: 'doc-100',
|
|
159
162
|
recordId: null,
|
|
163
|
+
updatedAt: Date.now(),
|
|
160
164
|
data: {
|
|
161
165
|
id: ObjectId.random(),
|
|
162
166
|
[ATTR_TYPE]: TYPE_PERSON,
|
|
@@ -168,6 +172,7 @@ describe('FtsIndex', () => {
|
|
|
168
172
|
queueId: null,
|
|
169
173
|
documentId: 'doc-200',
|
|
170
174
|
recordId: null,
|
|
175
|
+
updatedAt: Date.now(),
|
|
171
176
|
data: {
|
|
172
177
|
id: ObjectId.random(),
|
|
173
178
|
[ATTR_TYPE]: TYPE_PERSON,
|
|
@@ -179,6 +184,7 @@ describe('FtsIndex', () => {
|
|
|
179
184
|
queueId: null,
|
|
180
185
|
documentId: 'doc-1000',
|
|
181
186
|
recordId: null,
|
|
187
|
+
updatedAt: Date.now(),
|
|
182
188
|
data: {
|
|
183
189
|
id: ObjectId.random(),
|
|
184
190
|
[ATTR_TYPE]: TYPE_PERSON,
|
|
@@ -227,6 +233,7 @@ describe('FtsIndex', () => {
|
|
|
227
233
|
queueId: null,
|
|
228
234
|
documentId: 'doc-s1',
|
|
229
235
|
recordId: null,
|
|
236
|
+
updatedAt: Date.now(),
|
|
230
237
|
data: {
|
|
231
238
|
id: ObjectId.random(),
|
|
232
239
|
[ATTR_TYPE]: TYPE_PERSON,
|
|
@@ -239,6 +246,7 @@ describe('FtsIndex', () => {
|
|
|
239
246
|
queueId: null,
|
|
240
247
|
documentId: 'doc-s2',
|
|
241
248
|
recordId: null,
|
|
249
|
+
updatedAt: Date.now(),
|
|
242
250
|
data: {
|
|
243
251
|
id: ObjectId.random(),
|
|
244
252
|
[ATTR_TYPE]: TYPE_PERSON,
|
|
@@ -297,6 +305,7 @@ describe('FtsIndex', () => {
|
|
|
297
305
|
queueId: null,
|
|
298
306
|
documentId: 'doc-1',
|
|
299
307
|
recordId: null,
|
|
308
|
+
updatedAt: Date.now(),
|
|
300
309
|
data: {
|
|
301
310
|
id: ObjectId.random(),
|
|
302
311
|
[ATTR_TYPE]: TYPE_PERSON,
|
|
@@ -309,6 +318,7 @@ describe('FtsIndex', () => {
|
|
|
309
318
|
queueId: null,
|
|
310
319
|
documentId: 'doc-2',
|
|
311
320
|
recordId: null,
|
|
321
|
+
updatedAt: Date.now(),
|
|
312
322
|
data: {
|
|
313
323
|
id: ObjectId.random(),
|
|
314
324
|
[ATTR_TYPE]: TYPE_PERSON,
|
|
@@ -378,6 +388,7 @@ describe('FtsIndex', () => {
|
|
|
378
388
|
queueId: null,
|
|
379
389
|
documentId: 'doc-space',
|
|
380
390
|
recordId: null,
|
|
391
|
+
updatedAt: Date.now(),
|
|
381
392
|
data: {
|
|
382
393
|
id: ObjectId.random(),
|
|
383
394
|
[ATTR_TYPE]: TYPE_PERSON,
|
|
@@ -390,6 +401,7 @@ describe('FtsIndex', () => {
|
|
|
390
401
|
queueId: queue1,
|
|
391
402
|
documentId: null,
|
|
392
403
|
recordId: null,
|
|
404
|
+
updatedAt: Date.now(),
|
|
393
405
|
data: {
|
|
394
406
|
id: ObjectId.random(),
|
|
395
407
|
[ATTR_TYPE]: TYPE_PERSON,
|
|
@@ -402,6 +414,7 @@ describe('FtsIndex', () => {
|
|
|
402
414
|
queueId: queue2,
|
|
403
415
|
documentId: null,
|
|
404
416
|
recordId: null,
|
|
417
|
+
updatedAt: Date.now(),
|
|
405
418
|
data: {
|
|
406
419
|
id: ObjectId.random(),
|
|
407
420
|
[ATTR_TYPE]: TYPE_PERSON,
|
|
@@ -450,6 +463,7 @@ describe('FtsIndex', () => {
|
|
|
450
463
|
queueId: null,
|
|
451
464
|
documentId: 'doc-space',
|
|
452
465
|
recordId: null,
|
|
466
|
+
updatedAt: Date.now(),
|
|
453
467
|
data: {
|
|
454
468
|
id: ObjectId.random(),
|
|
455
469
|
[ATTR_TYPE]: TYPE_PERSON,
|
|
@@ -462,6 +476,7 @@ describe('FtsIndex', () => {
|
|
|
462
476
|
queueId,
|
|
463
477
|
documentId: null,
|
|
464
478
|
recordId: null,
|
|
479
|
+
updatedAt: Date.now(),
|
|
465
480
|
data: {
|
|
466
481
|
id: ObjectId.random(),
|
|
467
482
|
[ATTR_TYPE]: TYPE_PERSON,
|
|
@@ -511,6 +526,7 @@ describe('FtsIndex', () => {
|
|
|
511
526
|
queueId: null,
|
|
512
527
|
documentId: 'doc-s1',
|
|
513
528
|
recordId: null,
|
|
529
|
+
updatedAt: Date.now(),
|
|
514
530
|
data: {
|
|
515
531
|
id: ObjectId.random(),
|
|
516
532
|
[ATTR_TYPE]: TYPE_PERSON,
|
|
@@ -523,6 +539,7 @@ describe('FtsIndex', () => {
|
|
|
523
539
|
queueId: null,
|
|
524
540
|
documentId: 'doc-s2',
|
|
525
541
|
recordId: null,
|
|
542
|
+
updatedAt: Date.now(),
|
|
526
543
|
data: {
|
|
527
544
|
id: ObjectId.random(),
|
|
528
545
|
[ATTR_TYPE]: TYPE_PERSON,
|
|
@@ -535,6 +552,7 @@ describe('FtsIndex', () => {
|
|
|
535
552
|
queueId: queueInSpace2,
|
|
536
553
|
documentId: null,
|
|
537
554
|
recordId: null,
|
|
555
|
+
updatedAt: Date.now(),
|
|
538
556
|
data: {
|
|
539
557
|
id: ObjectId.random(),
|
|
540
558
|
[ATTR_TYPE]: TYPE_PERSON,
|
package/src/indexes/interface.ts
CHANGED
|
@@ -14,13 +14,13 @@ import { DXN, ObjectId, SpaceId } from '@dxos/keys';
|
|
|
14
14
|
import type { IndexerObject } from './interface';
|
|
15
15
|
import { ObjectMetaIndex } from './object-meta-index';
|
|
16
16
|
|
|
17
|
-
const TYPE_PERSON = DXN.parse('dxn:type:example.
|
|
18
|
-
const TYPE_PERSON_VERSIONLESS = DXN.parse('dxn:type:example.
|
|
19
|
-
const TYPE_RELATION = DXN.parse('dxn:type:example.
|
|
20
|
-
const TYPE_RELATION_UPDATED = DXN.parse('dxn:type:example.
|
|
21
|
-
const TYPE_WITH_UNDERSCORE = DXN.parse('dxn:type:example.
|
|
22
|
-
const TYPE_WITH_UNDERSCORE_VERSIONLESS = DXN.parse('dxn:type:example.
|
|
23
|
-
const TYPE_UNDERSCORE_FALSE_POSITIVE = DXN.parse('dxn:type:example.
|
|
17
|
+
const TYPE_PERSON = DXN.parse('dxn:type:com.example.type.person:0.1.0').toString();
|
|
18
|
+
const TYPE_PERSON_VERSIONLESS = DXN.parse('dxn:type:com.example.type.person').toString();
|
|
19
|
+
const TYPE_RELATION = DXN.parse('dxn:type:com.example.type.relation:0.1.0').toString();
|
|
20
|
+
const TYPE_RELATION_UPDATED = DXN.parse('dxn:type:com.example.type.relation-updated:0.1.0').toString();
|
|
21
|
+
const TYPE_WITH_UNDERSCORE = DXN.parse('dxn:type:com.example.type.person-extra:0.1.0').toString();
|
|
22
|
+
const TYPE_WITH_UNDERSCORE_VERSIONLESS = DXN.parse('dxn:type:com.example.type.person-extra').toString();
|
|
23
|
+
const TYPE_UNDERSCORE_FALSE_POSITIVE = DXN.parse('dxn:type:com.example.type.person-a-extra:0.1.0').toString();
|
|
24
24
|
|
|
25
25
|
const TestLayer = Layer.merge(
|
|
26
26
|
SqliteClient.layer({
|
|
@@ -43,6 +43,7 @@ describe('ObjectMetaIndex', () => {
|
|
|
43
43
|
queueId: ObjectId.random(),
|
|
44
44
|
documentId: null,
|
|
45
45
|
recordId: null,
|
|
46
|
+
updatedAt: Date.now(),
|
|
46
47
|
data: {
|
|
47
48
|
id: objectId,
|
|
48
49
|
[ATTR_TYPE]: TYPE_PERSON,
|
|
@@ -57,7 +58,7 @@ describe('ObjectMetaIndex', () => {
|
|
|
57
58
|
|
|
58
59
|
const otherTypeResults = yield* index.query({
|
|
59
60
|
spaceId,
|
|
60
|
-
typeDxn: DXN.parse('dxn:type:example.
|
|
61
|
+
typeDxn: DXN.parse('dxn:type:com.example.type.other').toString(),
|
|
61
62
|
});
|
|
62
63
|
expect(otherTypeResults).toEqual([]);
|
|
63
64
|
}).pipe(Effect.provide(TestLayer)),
|
|
@@ -77,6 +78,7 @@ describe('ObjectMetaIndex', () => {
|
|
|
77
78
|
queueId: ObjectId.random(),
|
|
78
79
|
documentId: null,
|
|
79
80
|
recordId: null,
|
|
81
|
+
updatedAt: Date.now(),
|
|
80
82
|
data: {
|
|
81
83
|
id: objectIdMatch,
|
|
82
84
|
[ATTR_TYPE]: TYPE_WITH_UNDERSCORE,
|
|
@@ -90,6 +92,7 @@ describe('ObjectMetaIndex', () => {
|
|
|
90
92
|
queueId: ObjectId.random(),
|
|
91
93
|
documentId: null,
|
|
92
94
|
recordId: null,
|
|
95
|
+
updatedAt: Date.now(),
|
|
93
96
|
data: {
|
|
94
97
|
id: objectIdFalsePositive,
|
|
95
98
|
[ATTR_TYPE]: TYPE_UNDERSCORE_FALSE_POSITIVE,
|
|
@@ -125,6 +128,7 @@ describe('ObjectMetaIndex', () => {
|
|
|
125
128
|
queueId: ObjectId.random(),
|
|
126
129
|
documentId: null,
|
|
127
130
|
recordId: null,
|
|
131
|
+
updatedAt: Date.now(),
|
|
128
132
|
data: {
|
|
129
133
|
id: objectId1,
|
|
130
134
|
[ATTR_TYPE]: TYPE_PERSON,
|
|
@@ -137,6 +141,7 @@ describe('ObjectMetaIndex', () => {
|
|
|
137
141
|
queueId: null,
|
|
138
142
|
documentId: 'doc-123',
|
|
139
143
|
recordId: null,
|
|
144
|
+
updatedAt: Date.now(),
|
|
140
145
|
data: {
|
|
141
146
|
id: objectId2,
|
|
142
147
|
[ATTR_TYPE]: TYPE_RELATION,
|
|
@@ -217,6 +222,7 @@ describe('ObjectMetaIndex', () => {
|
|
|
217
222
|
queueId: ObjectId.random(),
|
|
218
223
|
documentId: null,
|
|
219
224
|
recordId: null,
|
|
225
|
+
updatedAt: Date.now(),
|
|
220
226
|
data: {
|
|
221
227
|
id: objectId1,
|
|
222
228
|
[ATTR_TYPE]: TYPE_PERSON,
|
|
@@ -229,6 +235,7 @@ describe('ObjectMetaIndex', () => {
|
|
|
229
235
|
queueId: ObjectId.random(),
|
|
230
236
|
documentId: null,
|
|
231
237
|
recordId: null,
|
|
238
|
+
updatedAt: Date.now(),
|
|
232
239
|
data: {
|
|
233
240
|
id: objectId2,
|
|
234
241
|
[ATTR_TYPE]: TYPE_RELATION,
|
|
@@ -241,6 +248,7 @@ describe('ObjectMetaIndex', () => {
|
|
|
241
248
|
queueId: ObjectId.random(),
|
|
242
249
|
documentId: null,
|
|
243
250
|
recordId: null,
|
|
251
|
+
updatedAt: Date.now(),
|
|
244
252
|
data: {
|
|
245
253
|
id: relationId,
|
|
246
254
|
[ATTR_TYPE]: TYPE_RELATION,
|
|
@@ -319,4 +327,102 @@ describe('ObjectMetaIndex', () => {
|
|
|
319
327
|
expect(byTarget.map((_) => _.objectId)).toEqual([relationId]);
|
|
320
328
|
}).pipe(Effect.provide(TestLayer)),
|
|
321
329
|
);
|
|
330
|
+
|
|
331
|
+
it.effect('should set createdAt and updatedAt from source timestamp on insert and updatedAt on update', () =>
|
|
332
|
+
Effect.gen(function* () {
|
|
333
|
+
const index = new ObjectMetaIndex();
|
|
334
|
+
yield* index.migrate();
|
|
335
|
+
|
|
336
|
+
const spaceId = SpaceId.random();
|
|
337
|
+
const objectId = ObjectId.random();
|
|
338
|
+
const queueId = ObjectId.random();
|
|
339
|
+
|
|
340
|
+
const insertTimestamp = 1700000000000;
|
|
341
|
+
const item: IndexerObject = {
|
|
342
|
+
spaceId,
|
|
343
|
+
queueId,
|
|
344
|
+
documentId: null,
|
|
345
|
+
recordId: null,
|
|
346
|
+
updatedAt: insertTimestamp,
|
|
347
|
+
data: {
|
|
348
|
+
id: objectId,
|
|
349
|
+
[ATTR_TYPE]: TYPE_PERSON,
|
|
350
|
+
[ATTR_DELETED]: false,
|
|
351
|
+
},
|
|
352
|
+
};
|
|
353
|
+
|
|
354
|
+
yield* index.update([item]);
|
|
355
|
+
|
|
356
|
+
const results = yield* index.query({ spaceId, typeDxn: TYPE_PERSON });
|
|
357
|
+
expect(results.length).toBe(1);
|
|
358
|
+
expect(results[0].createdAt).toBe(insertTimestamp);
|
|
359
|
+
expect(results[0].updatedAt).toBe(insertTimestamp);
|
|
360
|
+
|
|
361
|
+
const updateTimestamp = 1700001000000;
|
|
362
|
+
yield* index.update([{ ...item, updatedAt: updateTimestamp, data: { ...item.data, [ATTR_DELETED]: true } }]);
|
|
363
|
+
|
|
364
|
+
const updated = yield* index.query({ spaceId, typeDxn: TYPE_PERSON });
|
|
365
|
+
expect(updated[0].createdAt).toBe(insertTimestamp);
|
|
366
|
+
expect(updated[0].updatedAt).toBe(updateTimestamp);
|
|
367
|
+
}).pipe(Effect.provide(TestLayer)),
|
|
368
|
+
);
|
|
369
|
+
|
|
370
|
+
it.effect('should query by time range', () =>
|
|
371
|
+
Effect.gen(function* () {
|
|
372
|
+
const index = new ObjectMetaIndex();
|
|
373
|
+
yield* index.migrate();
|
|
374
|
+
|
|
375
|
+
const spaceId = SpaceId.random();
|
|
376
|
+
const queueId1 = ObjectId.random();
|
|
377
|
+
const queueId2 = ObjectId.random();
|
|
378
|
+
const objectId1 = ObjectId.random();
|
|
379
|
+
const objectId2 = ObjectId.random();
|
|
380
|
+
|
|
381
|
+
const earlyTimestamp = 1700000000000;
|
|
382
|
+
const midpoint = 1700000500000;
|
|
383
|
+
const lateTimestamp = 1700001000000;
|
|
384
|
+
|
|
385
|
+
yield* index.update([
|
|
386
|
+
{
|
|
387
|
+
spaceId,
|
|
388
|
+
queueId: queueId1,
|
|
389
|
+
documentId: null,
|
|
390
|
+
recordId: null,
|
|
391
|
+
updatedAt: earlyTimestamp,
|
|
392
|
+
data: { id: objectId1, [ATTR_TYPE]: TYPE_PERSON, [ATTR_DELETED]: false },
|
|
393
|
+
},
|
|
394
|
+
]);
|
|
395
|
+
|
|
396
|
+
yield* index.update([
|
|
397
|
+
{
|
|
398
|
+
spaceId,
|
|
399
|
+
queueId: queueId2,
|
|
400
|
+
documentId: null,
|
|
401
|
+
recordId: null,
|
|
402
|
+
updatedAt: lateTimestamp,
|
|
403
|
+
data: { id: objectId2, [ATTR_TYPE]: TYPE_PERSON, [ATTR_DELETED]: false },
|
|
404
|
+
},
|
|
405
|
+
]);
|
|
406
|
+
|
|
407
|
+
// Query all -- both should match.
|
|
408
|
+
const all = yield* index.queryByTimeRange({ spaceIds: [spaceId], includeAllQueues: true });
|
|
409
|
+
expect(all.map((_) => _.objectId).sort()).toEqual([objectId1, objectId2].sort());
|
|
410
|
+
|
|
411
|
+
// Query only objects updated after midpoint (object2 has lateTimestamp).
|
|
412
|
+
const afterMid = yield* index.queryByTimeRange({
|
|
413
|
+
spaceIds: [spaceId],
|
|
414
|
+
updatedAfter: midpoint,
|
|
415
|
+
includeAllQueues: true,
|
|
416
|
+
});
|
|
417
|
+
expect(afterMid.map((_) => _.objectId)).toEqual([objectId2]);
|
|
418
|
+
|
|
419
|
+
// Query only objects created before midpoint (object1 has earlyTimestamp).
|
|
420
|
+
const beforeMid = yield* index.queryByTimeRange({
|
|
421
|
+
spaceIds: [spaceId],
|
|
422
|
+
createdBefore: midpoint,
|
|
423
|
+
includeAllQueues: true,
|
|
424
|
+
});
|
|
425
|
+
expect(beforeMid.map((_) => _.objectId)).toEqual([objectId1]);
|
|
426
|
+
}).pipe(Effect.provide(TestLayer)),
|
|
427
|
+
);
|
|
322
428
|
});
|