@dxos/index-core 0.8.4-main.c85a9c8dae → 0.8.4-main.d05539e30a
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +102 -5
- package/README.md +1 -1
- package/dist/lib/neutral/index.mjs +155 -59
- package/dist/lib/neutral/index.mjs.map +3 -3
- package/dist/lib/neutral/meta.json +1 -1
- package/dist/types/src/index-engine.d.ts +29 -9
- package/dist/types/src/index-engine.d.ts.map +1 -1
- package/dist/types/src/index-tracker.d.ts +3 -3
- package/dist/types/src/index-tracker.d.ts.map +1 -1
- package/dist/types/src/index.d.ts +1 -1
- package/dist/types/src/index.d.ts.map +1 -1
- package/dist/types/src/indexes/fts-index.d.ts +2 -1
- package/dist/types/src/indexes/fts-index.d.ts.map +1 -1
- package/dist/types/src/indexes/interface.d.ts +9 -0
- package/dist/types/src/indexes/interface.d.ts.map +1 -1
- package/dist/types/src/indexes/object-meta-index.d.ts +28 -6
- package/dist/types/src/indexes/object-meta-index.d.ts.map +1 -1
- package/dist/types/src/indexes/reverse-ref-index.d.ts +2 -2
- package/dist/types/src/indexes/reverse-ref-index.d.ts.map +1 -1
- package/dist/types/tsconfig.tsbuildinfo +1 -1
- package/package.json +13 -13
- package/src/index-engine.test.ts +168 -8
- package/src/index-engine.ts +104 -28
- package/src/index-tracker.ts +9 -0
- package/src/index.ts +7 -1
- package/src/indexes/fts-index.test.ts +153 -3
- package/src/indexes/fts-index.ts +25 -8
- package/src/indexes/interface.ts +10 -0
- package/src/indexes/object-meta-index.test.ts +170 -15
- package/src/indexes/object-meta-index.ts +107 -25
- package/src/indexes/reverse-ref-index.test.ts +31 -17
- package/src/indexes/reverse-ref-index.ts +11 -12
package/LICENSE
CHANGED
|
@@ -1,8 +1,105 @@
|
|
|
1
|
-
|
|
2
|
-
Copyright (c) 2022 DXOS
|
|
1
|
+
# Functional Source License, Version 1.1, ALv2 Future License
|
|
3
2
|
|
|
4
|
-
|
|
3
|
+
## Abbreviation
|
|
5
4
|
|
|
6
|
-
|
|
5
|
+
FSL-1.1-Apache-2.0
|
|
7
6
|
|
|
8
|
-
|
|
7
|
+
## Notice
|
|
8
|
+
|
|
9
|
+
Copyright 2026 DXOS
|
|
10
|
+
|
|
11
|
+
## Terms and Conditions
|
|
12
|
+
|
|
13
|
+
### Licensor ("We")
|
|
14
|
+
|
|
15
|
+
The party offering the Software under these Terms and Conditions.
|
|
16
|
+
|
|
17
|
+
### The Software
|
|
18
|
+
|
|
19
|
+
The "Software" is each version of the software that we make available under
|
|
20
|
+
these Terms and Conditions, as indicated by our inclusion of these Terms and
|
|
21
|
+
Conditions with the Software.
|
|
22
|
+
|
|
23
|
+
### License Grant
|
|
24
|
+
|
|
25
|
+
Subject to your compliance with this License Grant and the Patents,
|
|
26
|
+
Redistribution and Trademark clauses below, we hereby grant you the right to
|
|
27
|
+
use, copy, modify, create derivative works, publicly perform, publicly display
|
|
28
|
+
and redistribute the Software for any Permitted Purpose identified below.
|
|
29
|
+
|
|
30
|
+
### Permitted Purpose
|
|
31
|
+
|
|
32
|
+
A Permitted Purpose is any purpose other than a Competing Use. A Competing Use
|
|
33
|
+
means making the Software available to others in a commercial product or
|
|
34
|
+
service that:
|
|
35
|
+
|
|
36
|
+
1. substitutes for the Software;
|
|
37
|
+
|
|
38
|
+
2. substitutes for any other product or service we offer using the Software
|
|
39
|
+
that exists as of the date we make the Software available; or
|
|
40
|
+
|
|
41
|
+
3. offers the same or substantially similar functionality as the Software.
|
|
42
|
+
|
|
43
|
+
Permitted Purposes specifically include using the Software:
|
|
44
|
+
|
|
45
|
+
1. for your internal use and access;
|
|
46
|
+
|
|
47
|
+
2. for non-commercial education;
|
|
48
|
+
|
|
49
|
+
3. for non-commercial research; and
|
|
50
|
+
|
|
51
|
+
4. in connection with professional services that you provide to a licensee
|
|
52
|
+
using the Software in accordance with these Terms and Conditions.
|
|
53
|
+
|
|
54
|
+
### Patents
|
|
55
|
+
|
|
56
|
+
To the extent your use for a Permitted Purpose would necessarily infringe our
|
|
57
|
+
patents, the license grant above includes a license under our patents. If you
|
|
58
|
+
make a claim against any party that the Software infringes or contributes to
|
|
59
|
+
the infringement of any patent, then your patent license to the Software ends
|
|
60
|
+
immediately.
|
|
61
|
+
|
|
62
|
+
### Redistribution
|
|
63
|
+
|
|
64
|
+
The Terms and Conditions apply to all copies, modifications and derivatives of
|
|
65
|
+
the Software.
|
|
66
|
+
|
|
67
|
+
If you redistribute any copies, modifications or derivatives of the Software,
|
|
68
|
+
you must include a copy of or a link to these Terms and Conditions and not
|
|
69
|
+
remove any copyright notices provided in or with the Software.
|
|
70
|
+
|
|
71
|
+
### Disclaimer
|
|
72
|
+
|
|
73
|
+
THE SOFTWARE IS PROVIDED "AS IS" AND WITHOUT WARRANTIES OF ANY KIND, EXPRESS OR
|
|
74
|
+
IMPLIED, INCLUDING WITHOUT LIMITATION WARRANTIES OF FITNESS FOR A PARTICULAR
|
|
75
|
+
PURPOSE, MERCHANTABILITY, TITLE OR NON-INFRINGEMENT.
|
|
76
|
+
|
|
77
|
+
IN NO EVENT WILL WE HAVE ANY LIABILITY TO YOU ARISING OUT OF OR RELATED TO THE
|
|
78
|
+
SOFTWARE, INCLUDING INDIRECT, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES,
|
|
79
|
+
EVEN IF WE HAVE BEEN INFORMED OF THEIR POSSIBILITY IN ADVANCE.
|
|
80
|
+
|
|
81
|
+
### Trademarks
|
|
82
|
+
|
|
83
|
+
Except for displaying the License Details and identifying us as the origin of
|
|
84
|
+
the Software, you have no right under these Terms and Conditions to use our
|
|
85
|
+
trademarks, trade names, service marks or product names.
|
|
86
|
+
|
|
87
|
+
## Grant of Future License
|
|
88
|
+
|
|
89
|
+
We hereby irrevocably grant you an additional license to use the Software under
|
|
90
|
+
the Apache License, Version 2.0 that is effective on the second anniversary of
|
|
91
|
+
the date we make the Software available. On or after that date, you may use the
|
|
92
|
+
Software under the Apache License, Version 2.0, in which case the following
|
|
93
|
+
will apply:
|
|
94
|
+
|
|
95
|
+
Licensed under the Apache License, Version 2.0 (the "License"); you may not use
|
|
96
|
+
this file except in compliance with the License.
|
|
97
|
+
|
|
98
|
+
You may obtain a copy of the License at
|
|
99
|
+
|
|
100
|
+
http://www.apache.org/licenses/LICENSE-2.0
|
|
101
|
+
|
|
102
|
+
Unless required by applicable law or agreed to in writing, software distributed
|
|
103
|
+
under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
|
|
104
|
+
CONDITIONS OF ANY KIND, either express or implied. See the License for the
|
|
105
|
+
specific language governing permissions and limitations under the License.
|
package/README.md
CHANGED
|
@@ -18,4 +18,4 @@ pnpm i @dxos/index-core
|
|
|
18
18
|
|
|
19
19
|
Your ideas, issues, and code are most welcome. Please take a look at our [community code of conduct](https://github.com/dxos/dxos/blob/main/CODE_OF_CONDUCT.md), the [issue guide](https://github.com/dxos/dxos/blob/main/CONTRIBUTING.md#submitting-issues), and the [PR contribution guide](https://github.com/dxos/dxos/blob/main/CONTRIBUTING.md#submitting-prs).
|
|
20
20
|
|
|
21
|
-
License: [
|
|
21
|
+
License: [FSL-1.1-Apache-2.0](./LICENSE) Copyright 2022 © DXOS
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
// src/index-engine.ts
|
|
2
2
|
import * as Effect5 from "effect/Effect";
|
|
3
|
+
import { ATTR_TYPE as ATTR_TYPE2 } from "@dxos/echo/internal";
|
|
3
4
|
import * as SqlTransaction from "@dxos/sql-sqlite/SqlTransaction";
|
|
4
5
|
|
|
5
6
|
// src/index-tracker.ts
|
|
@@ -31,6 +32,9 @@ var IndexCursor = Schema.Struct({
|
|
|
31
32
|
*/
|
|
32
33
|
cursor: Schema.Union(Schema.Number, Schema.String)
|
|
33
34
|
});
|
|
35
|
+
var DEPRECATED_INDEX_NAMES = [
|
|
36
|
+
"fts"
|
|
37
|
+
];
|
|
34
38
|
var IndexTracker = class {
|
|
35
39
|
migrate = Effect.fn("IndexTracker.migrate")(function* () {
|
|
36
40
|
const sql = yield* SqlClient.SqlClient;
|
|
@@ -42,6 +46,9 @@ var IndexTracker = class {
|
|
|
42
46
|
cursor,
|
|
43
47
|
PRIMARY KEY (indexName, spaceId, sourceName, resourceId)
|
|
44
48
|
)`;
|
|
49
|
+
yield* Effect.forEach(DEPRECATED_INDEX_NAMES, (indexName) => {
|
|
50
|
+
return sql`DELETE FROM indexCursor WHERE indexName = ${indexName}`;
|
|
51
|
+
});
|
|
45
52
|
});
|
|
46
53
|
queryCursors = Effect.fn("IndexTracker.queryCursors")((query) => Effect.gen(function* () {
|
|
47
54
|
const sql = yield* SqlClient.SqlClient;
|
|
@@ -82,6 +89,7 @@ var IndexTracker = class {
|
|
|
82
89
|
// src/indexes/fts-index.ts
|
|
83
90
|
import * as SqlClient3 from "@effect/sql/SqlClient";
|
|
84
91
|
import * as Effect2 from "effect/Effect";
|
|
92
|
+
var SQL_CHUNK_SIZE = 500;
|
|
85
93
|
var escapeFts5Query = (text) => {
|
|
86
94
|
return text.split(/\s+/).filter(Boolean).map((term) => `"${term.replace(/"/g, '""')}"`).join(" ");
|
|
87
95
|
};
|
|
@@ -143,6 +151,7 @@ var FtsIndex = class {
|
|
|
143
151
|
/**
|
|
144
152
|
* Query snapshots by recordIds.
|
|
145
153
|
* Returns the parsed JSON snapshots for queue objects.
|
|
154
|
+
* RecordIds not present in the FTS index are silently omitted from the result.
|
|
146
155
|
*/
|
|
147
156
|
querySnapshotsJSON(recordIds) {
|
|
148
157
|
return Effect2.gen(function* () {
|
|
@@ -150,11 +159,21 @@ var FtsIndex = class {
|
|
|
150
159
|
return [];
|
|
151
160
|
}
|
|
152
161
|
const sql = yield* SqlClient3.SqlClient;
|
|
153
|
-
const
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
162
|
+
const chunks = [];
|
|
163
|
+
for (let i = 0; i < recordIds.length; i += SQL_CHUNK_SIZE) {
|
|
164
|
+
chunks.push(recordIds.slice(i, i + SQL_CHUNK_SIZE));
|
|
165
|
+
}
|
|
166
|
+
const allResults = [];
|
|
167
|
+
for (const chunk of chunks) {
|
|
168
|
+
const rows = yield* sql`SELECT rowid, snapshot FROM ftsIndex WHERE rowid IN ${sql.in(chunk)}`;
|
|
169
|
+
for (const r of rows) {
|
|
170
|
+
allResults.push({
|
|
171
|
+
recordId: r.rowid,
|
|
172
|
+
snapshot: JSON.parse(r.snapshot)
|
|
173
|
+
});
|
|
174
|
+
}
|
|
175
|
+
}
|
|
176
|
+
return allResults;
|
|
158
177
|
});
|
|
159
178
|
}
|
|
160
179
|
update = Effect2.fn("FtsIndex.update")((objects) => Effect2.gen(function* () {
|
|
@@ -190,18 +209,24 @@ var ObjectMeta = Schema2.Struct({
|
|
|
190
209
|
recordId: Schema2.Number,
|
|
191
210
|
objectId: Schema2.String,
|
|
192
211
|
queueId: Schema2.String,
|
|
212
|
+
/** Queue subspace namespace (e.g. 'data', 'trace'). Empty string for non-queue objects. */
|
|
213
|
+
queueNamespace: Schema2.String,
|
|
193
214
|
spaceId: Schema2.String,
|
|
194
215
|
documentId: Schema2.String,
|
|
195
216
|
entityKind: Schema2.String,
|
|
196
217
|
/** The versioned DXN of the type of the object. */
|
|
197
|
-
|
|
218
|
+
typeDXN: Schema2.String,
|
|
198
219
|
deleted: Schema2.Boolean,
|
|
199
220
|
source: Schema2.NullOr(Schema2.String),
|
|
200
221
|
target: Schema2.NullOr(Schema2.String),
|
|
201
222
|
/** Parent object id (nullable). */
|
|
202
223
|
parent: Schema2.NullOr(Schema2.String),
|
|
203
224
|
/** Monotonically increasing sequence number assigned on insert/update for tracking indexing order. */
|
|
204
|
-
version: Schema2.Number
|
|
225
|
+
version: Schema2.Number,
|
|
226
|
+
/** Unix ms timestamp when the object was first indexed. */
|
|
227
|
+
createdAt: Schema2.NullOr(Schema2.Number),
|
|
228
|
+
/** Unix ms timestamp when the object was last re-indexed. */
|
|
229
|
+
updatedAt: Schema2.NullOr(Schema2.Number)
|
|
205
230
|
});
|
|
206
231
|
var buildSourceCondition = (sql, spaceIds, includeAllQueues, queueIds) => {
|
|
207
232
|
const conditions = [];
|
|
@@ -227,26 +252,34 @@ var ObjectMetaIndex = class {
|
|
|
227
252
|
recordId INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
228
253
|
objectId TEXT NOT NULL,
|
|
229
254
|
queueId TEXT NOT NULL DEFAULT '',
|
|
255
|
+
queueNamespace TEXT NOT NULL DEFAULT '',
|
|
230
256
|
spaceId TEXT NOT NULL,
|
|
231
257
|
documentId TEXT NOT NULL DEFAULT '',
|
|
232
258
|
entityKind TEXT NOT NULL,
|
|
233
|
-
|
|
259
|
+
typeDXN TEXT NOT NULL,
|
|
234
260
|
deleted INTEGER NOT NULL,
|
|
235
261
|
source TEXT,
|
|
236
262
|
target TEXT,
|
|
237
263
|
parent TEXT,
|
|
238
|
-
version INTEGER NOT NULL
|
|
264
|
+
version INTEGER NOT NULL,
|
|
265
|
+
createdAt INTEGER,
|
|
266
|
+
updatedAt INTEGER
|
|
239
267
|
)`;
|
|
240
268
|
yield* Effect3.catchAll(sql`ALTER TABLE objectMeta ADD COLUMN parent TEXT`, () => Effect3.void);
|
|
269
|
+
yield* Effect3.catchAll(sql`ALTER TABLE objectMeta ADD COLUMN createdAt INTEGER`, () => Effect3.void);
|
|
270
|
+
yield* Effect3.catchAll(sql`ALTER TABLE objectMeta ADD COLUMN updatedAt INTEGER`, () => Effect3.void);
|
|
271
|
+
yield* Effect3.catchAll(sql`ALTER TABLE objectMeta ADD COLUMN queueNamespace TEXT NOT NULL DEFAULT ''`, () => Effect3.void);
|
|
241
272
|
yield* sql`CREATE INDEX IF NOT EXISTS idx_object_index_objectId ON objectMeta(spaceId, objectId)`;
|
|
242
|
-
yield* sql`CREATE INDEX IF NOT EXISTS
|
|
273
|
+
yield* sql`CREATE INDEX IF NOT EXISTS idx_object_index_typeDXN ON objectMeta(spaceId, typeDXN)`;
|
|
243
274
|
yield* sql`CREATE INDEX IF NOT EXISTS idx_object_index_version ON objectMeta(version)`;
|
|
244
275
|
yield* sql`CREATE INDEX IF NOT EXISTS idx_object_index_parent ON objectMeta(spaceId, parent)`;
|
|
276
|
+
yield* sql`CREATE INDEX IF NOT EXISTS idx_object_index_updatedAt ON objectMeta(updatedAt)`;
|
|
277
|
+
yield* sql`CREATE INDEX IF NOT EXISTS idx_object_index_createdAt ON objectMeta(createdAt)`;
|
|
245
278
|
});
|
|
246
279
|
query = Effect3.fn("ObjectMetaIndex.query")((query) => Effect3.gen(function* () {
|
|
247
280
|
const sql = yield* SqlClient5.SqlClient;
|
|
248
|
-
const parsedType = DXN.tryParse(query.
|
|
249
|
-
const rows = parsedType && parsedType.version === void 0 ? yield* sql`SELECT * FROM objectMeta WHERE spaceId = ${query.spaceId} AND (
|
|
281
|
+
const parsedType = DXN.tryParse(query.typeDXN)?.asTypeDXN();
|
|
282
|
+
const rows = parsedType && parsedType.version === void 0 ? yield* sql`SELECT * FROM objectMeta WHERE spaceId = ${query.spaceId} AND (typeDXN = ${query.typeDXN} OR typeDXN LIKE ${_escapeLikePrefix(query.typeDXN)} ESCAPE '\\')` : yield* sql`SELECT * FROM objectMeta WHERE spaceId = ${query.spaceId} AND typeDXN = ${query.typeDXN}`;
|
|
250
283
|
return rows.map((row) => ({
|
|
251
284
|
...row,
|
|
252
285
|
deleted: !!row.deleted
|
|
@@ -282,12 +315,12 @@ var ObjectMetaIndex = class {
|
|
|
282
315
|
}
|
|
283
316
|
const sql = yield* SqlClient5.SqlClient;
|
|
284
317
|
const sourceCondition = buildSourceCondition(sql, spaceIds, includeAllQueues, queueIds);
|
|
285
|
-
const typeWhere = sql.or(typeDxns.map((
|
|
286
|
-
const parsedType = DXN.tryParse(
|
|
318
|
+
const typeWhere = sql.or(typeDxns.map((typeDXN) => {
|
|
319
|
+
const parsedType = DXN.tryParse(typeDXN)?.asTypeDXN();
|
|
287
320
|
return parsedType && parsedType.version === void 0 ? sql.or([
|
|
288
|
-
sql`
|
|
289
|
-
sql`
|
|
290
|
-
]) : sql`
|
|
321
|
+
sql`typeDXN = ${typeDXN}`,
|
|
322
|
+
sql`typeDXN LIKE ${_escapeLikePrefix(typeDXN)} ESCAPE '\\'`
|
|
323
|
+
]) : sql`typeDXN = ${typeDXN}`;
|
|
291
324
|
}));
|
|
292
325
|
const rows = inverted ? yield* sql`SELECT * FROM objectMeta WHERE ${sourceCondition} AND NOT ${typeWhere}` : yield* sql`SELECT * FROM objectMeta WHERE ${sourceCondition} AND ${typeWhere}`;
|
|
293
326
|
return rows.map((row) => ({
|
|
@@ -311,7 +344,7 @@ var ObjectMetaIndex = class {
|
|
|
311
344
|
update = Effect3.fn("ObjectMetaIndex.update")((objects) => Effect3.gen(function* () {
|
|
312
345
|
const sql = yield* SqlClient5.SqlClient;
|
|
313
346
|
yield* Effect3.forEach(objects, (object) => Effect3.gen(function* () {
|
|
314
|
-
const { spaceId, queueId, documentId, data } = object;
|
|
347
|
+
const { spaceId, queueId, queueNamespace, documentId, data } = object;
|
|
315
348
|
const castData = data;
|
|
316
349
|
const objectId = castData.id;
|
|
317
350
|
let existing;
|
|
@@ -326,32 +359,37 @@ var ObjectMetaIndex = class {
|
|
|
326
359
|
const [{ v }] = result;
|
|
327
360
|
const version = (v ?? 0) + 1;
|
|
328
361
|
const entityKind = castData[ATTR_RELATION_SOURCE] ? "relation" : "object";
|
|
329
|
-
const
|
|
362
|
+
const typeDXN = castData[ATTR_TYPE] ? String(castData[ATTR_TYPE]) : "type";
|
|
330
363
|
const deleted = castData[ATTR_DELETED] ? 1 : 0;
|
|
331
364
|
const source = entityKind === "relation" ? castData[ATTR_RELATION_SOURCE] ?? null : null;
|
|
332
365
|
const target = entityKind === "relation" ? castData[ATTR_RELATION_TARGET] ?? null : null;
|
|
333
366
|
const parent = castData[ATTR_PARENT] ?? null;
|
|
367
|
+
const sourceTimestamp = object.updatedAt;
|
|
334
368
|
if (existing.length > 0) {
|
|
335
369
|
yield* sql`
|
|
336
370
|
UPDATE objectMeta SET
|
|
337
371
|
version = ${version},
|
|
372
|
+
queueNamespace = ${queueNamespace ?? ""},
|
|
338
373
|
entityKind = ${entityKind},
|
|
339
|
-
|
|
374
|
+
typeDXN = ${typeDXN},
|
|
340
375
|
deleted = ${deleted},
|
|
341
376
|
source = ${source},
|
|
342
377
|
target = ${target},
|
|
343
|
-
parent = ${parent}
|
|
378
|
+
parent = ${parent},
|
|
379
|
+
updatedAt = ${sourceTimestamp}
|
|
344
380
|
WHERE recordId = ${existing[0].recordId}
|
|
345
381
|
`;
|
|
346
382
|
} else {
|
|
347
383
|
yield* sql`
|
|
348
384
|
INSERT INTO objectMeta (
|
|
349
|
-
objectId, queueId, spaceId, documentId,
|
|
350
|
-
entityKind,
|
|
385
|
+
objectId, queueId, queueNamespace, spaceId, documentId,
|
|
386
|
+
entityKind, typeDXN, deleted, source, target, parent, version,
|
|
387
|
+
createdAt, updatedAt
|
|
351
388
|
) VALUES (
|
|
352
|
-
${objectId}, ${queueId ?? ""}, ${spaceId}, ${documentId ?? ""},
|
|
353
|
-
${entityKind}, ${
|
|
354
|
-
${source}, ${target}, ${parent}, ${version}
|
|
389
|
+
${objectId}, ${queueId ?? ""}, ${queueNamespace ?? ""}, ${spaceId}, ${documentId ?? ""},
|
|
390
|
+
${entityKind}, ${typeDXN}, ${deleted},
|
|
391
|
+
${source}, ${target}, ${parent}, ${version},
|
|
392
|
+
${sourceTimestamp}, ${sourceTimestamp}
|
|
355
393
|
)
|
|
356
394
|
`;
|
|
357
395
|
}
|
|
@@ -411,14 +449,47 @@ var ObjectMetaIndex = class {
|
|
|
411
449
|
};
|
|
412
450
|
}));
|
|
413
451
|
/**
|
|
452
|
+
* Query objects by timestamp range.
|
|
453
|
+
*/
|
|
454
|
+
queryByTimeRange = Effect3.fn("ObjectMetaIndex.queryByTimeRange")((query) => Effect3.gen(function* () {
|
|
455
|
+
if (query.spaceIds.length === 0 && (!query.queueIds || query.queueIds.length === 0)) {
|
|
456
|
+
return [];
|
|
457
|
+
}
|
|
458
|
+
const sql = yield* SqlClient5.SqlClient;
|
|
459
|
+
const sourceCondition = buildSourceCondition(sql, query.spaceIds, query.includeAllQueues ?? false, query.queueIds ?? null);
|
|
460
|
+
const timeConditions = [];
|
|
461
|
+
if (query.updatedAfter != null) {
|
|
462
|
+
timeConditions.push(sql`updatedAt >= ${query.updatedAfter}`);
|
|
463
|
+
}
|
|
464
|
+
if (query.updatedBefore != null) {
|
|
465
|
+
timeConditions.push(sql`updatedAt <= ${query.updatedBefore}`);
|
|
466
|
+
}
|
|
467
|
+
if (query.createdAfter != null) {
|
|
468
|
+
timeConditions.push(sql`createdAt >= ${query.createdAfter}`);
|
|
469
|
+
}
|
|
470
|
+
if (query.createdBefore != null) {
|
|
471
|
+
timeConditions.push(sql`createdAt <= ${query.createdBefore}`);
|
|
472
|
+
}
|
|
473
|
+
const rows = timeConditions.length > 0 ? yield* sql`SELECT * FROM objectMeta WHERE ${sourceCondition} AND ${sql.and(timeConditions)}` : yield* sql`SELECT * FROM objectMeta WHERE ${sourceCondition}`;
|
|
474
|
+
return rows.map((row) => ({
|
|
475
|
+
...row,
|
|
476
|
+
deleted: !!row.deleted
|
|
477
|
+
}));
|
|
478
|
+
}));
|
|
479
|
+
/**
|
|
414
480
|
* Query children by parent object ids.
|
|
481
|
+
* Matches both:
|
|
482
|
+
* - Objects whose `parent` field references one of the given parent ids (standard parent/child hierarchy).
|
|
483
|
+
* - Queue items whose `queueId` equals one of the parent ids (e.g. items inside a Feed, since a feed's queue
|
|
484
|
+
* DXN uses the feed's object id as its queue id — see `Feed.getQueueDxn`).
|
|
415
485
|
*/
|
|
416
486
|
queryChildren = Effect3.fn("ObjectMetaIndex.queryChildren")((query) => Effect3.gen(function* () {
|
|
417
487
|
if (query.parentIds.length === 0) {
|
|
418
488
|
return [];
|
|
419
489
|
}
|
|
420
490
|
const sql = yield* SqlClient5.SqlClient;
|
|
421
|
-
const
|
|
491
|
+
const parentDxns = query.parentIds.map((id) => DXN.fromLocalObjectId(id).toString());
|
|
492
|
+
const rows = yield* sql`SELECT * FROM objectMeta WHERE ${sql.in("spaceId", query.spaceId)} AND (${sql.in("parent", parentDxns)} OR ${sql.in("queueId", query.parentIds)})`;
|
|
422
493
|
return rows.map((row) => ({
|
|
423
494
|
...row,
|
|
424
495
|
deleted: !!row.deleted
|
|
@@ -447,15 +518,7 @@ var EscapedPropPath = class extends Schema3.String.annotations({
|
|
|
447
518
|
let current = "";
|
|
448
519
|
for (let i = 0; i < path.length; i++) {
|
|
449
520
|
if (path[i] === "\\") {
|
|
450
|
-
invariant(i + 1 < path.length && (path[i + 1] === "." || path[i + 1] === "\\"), "Malformed escaping.", {
|
|
451
|
-
F: __dxlog_file,
|
|
452
|
-
L: 34,
|
|
453
|
-
S: this,
|
|
454
|
-
A: [
|
|
455
|
-
"i + 1 < path.length && (path[i + 1] === '.' || path[i + 1] === '\\\\')",
|
|
456
|
-
"'Malformed escaping.'"
|
|
457
|
-
]
|
|
458
|
-
});
|
|
521
|
+
invariant(i + 1 < path.length && (path[i + 1] === "." || path[i + 1] === "\\"), "Malformed escaping.", { "~LogMeta": "~LogMeta", F: __dxlog_file, L: 25, S: this, A: ["i + 1 < path.length && (path[i + 1] === '.' || path[i + 1] === '\\\\')", "'Malformed escaping.'"] });
|
|
459
522
|
current = current + path[i + 1];
|
|
460
523
|
i++;
|
|
461
524
|
} else if (path[i] === ".") {
|
|
@@ -482,7 +545,7 @@ var extractReferences = (data) => {
|
|
|
482
545
|
}
|
|
483
546
|
refs.push({
|
|
484
547
|
path,
|
|
485
|
-
|
|
548
|
+
targetDXN: dxn.toString()
|
|
486
549
|
});
|
|
487
550
|
} else if (typeof value === "object" && value !== null && !Array.isArray(value)) {
|
|
488
551
|
for (const [key, v] of Object.entries(value)) {
|
|
@@ -505,7 +568,7 @@ var extractReferences = (data) => {
|
|
|
505
568
|
};
|
|
506
569
|
var ReverseRef = Schema4.Struct({
|
|
507
570
|
recordId: Schema4.Number,
|
|
508
|
-
|
|
571
|
+
targetDXN: Schema4.String,
|
|
509
572
|
/**
|
|
510
573
|
* Escaped property path within an object.
|
|
511
574
|
*
|
|
@@ -522,18 +585,18 @@ var ReverseRefIndex = class {
|
|
|
522
585
|
const sql = yield* SqlClient7.SqlClient;
|
|
523
586
|
yield* sql`CREATE TABLE IF NOT EXISTS reverseRef (
|
|
524
587
|
recordId INTEGER NOT NULL,
|
|
525
|
-
|
|
588
|
+
targetDXN TEXT NOT NULL,
|
|
526
589
|
propPath TEXT NOT NULL,
|
|
527
|
-
PRIMARY KEY (recordId,
|
|
590
|
+
PRIMARY KEY (recordId, targetDXN, propPath)
|
|
528
591
|
)`;
|
|
529
|
-
yield* sql`CREATE INDEX IF NOT EXISTS idx_reverse_ref_target ON reverseRef(
|
|
592
|
+
yield* sql`CREATE INDEX IF NOT EXISTS idx_reverse_ref_target ON reverseRef(targetDXN)`;
|
|
530
593
|
});
|
|
531
594
|
/**
|
|
532
595
|
* Query all references pointing to a target DXN.
|
|
533
596
|
*/
|
|
534
|
-
query = Effect4.fn("ReverseRefIndex.query")(({
|
|
597
|
+
query = Effect4.fn("ReverseRefIndex.query")(({ targetDXN }) => Effect4.gen(function* () {
|
|
535
598
|
const sql = yield* SqlClient7.SqlClient;
|
|
536
|
-
const rows = yield* sql`SELECT * FROM reverseRef WHERE
|
|
599
|
+
const rows = yield* sql`SELECT * FROM reverseRef WHERE targetDXN = ${targetDXN}`;
|
|
537
600
|
return rows;
|
|
538
601
|
}));
|
|
539
602
|
update = Effect4.fn("ReverseRefIndex.update")((objects) => Effect4.gen(function* () {
|
|
@@ -545,7 +608,7 @@ var ReverseRefIndex = class {
|
|
|
545
608
|
}
|
|
546
609
|
yield* sql`DELETE FROM reverseRef WHERE recordId = ${recordId}`;
|
|
547
610
|
const refs = extractReferences(data);
|
|
548
|
-
yield* Effect4.forEach(refs, (ref) => sql`INSERT INTO reverseRef (recordId,
|
|
611
|
+
yield* Effect4.forEach(refs, (ref) => sql`INSERT INTO reverseRef (recordId, targetDXN, propPath) VALUES (${recordId}, ${ref.targetDXN}, ${EscapedPropPath.escape(ref.path)})`, {
|
|
549
612
|
discard: true
|
|
550
613
|
});
|
|
551
614
|
}), {
|
|
@@ -555,6 +618,33 @@ var ReverseRefIndex = class {
|
|
|
555
618
|
};
|
|
556
619
|
|
|
557
620
|
// src/index-engine.ts
|
|
621
|
+
var makeEmptyIndexingResult = () => ({
|
|
622
|
+
updated: 0,
|
|
623
|
+
done: true,
|
|
624
|
+
spaces: /* @__PURE__ */ new Set(),
|
|
625
|
+
queues: /* @__PURE__ */ new Set(),
|
|
626
|
+
documents: /* @__PURE__ */ new Set(),
|
|
627
|
+
types: /* @__PURE__ */ new Set(),
|
|
628
|
+
objects: /* @__PURE__ */ new Set()
|
|
629
|
+
});
|
|
630
|
+
var accumulateIndexingResult = (acc, objects) => {
|
|
631
|
+
for (const obj of objects) {
|
|
632
|
+
acc.spaces.add(obj.spaceId);
|
|
633
|
+
if (obj.queueId) {
|
|
634
|
+
acc.queues.add(obj.queueId);
|
|
635
|
+
}
|
|
636
|
+
if (obj.documentId) {
|
|
637
|
+
acc.documents.add(obj.documentId);
|
|
638
|
+
}
|
|
639
|
+
const t = obj.data[ATTR_TYPE2];
|
|
640
|
+
if (t) {
|
|
641
|
+
acc.types.add(String(t));
|
|
642
|
+
}
|
|
643
|
+
if (obj.data.id) {
|
|
644
|
+
acc.objects.add(obj.data.id);
|
|
645
|
+
}
|
|
646
|
+
}
|
|
647
|
+
};
|
|
558
648
|
var IndexEngine = class {
|
|
559
649
|
#tracker;
|
|
560
650
|
#objectMetaIndex;
|
|
@@ -607,6 +697,9 @@ var IndexEngine = class {
|
|
|
607
697
|
queryTypes(query) {
|
|
608
698
|
return this.#objectMetaIndex.queryTypes(query);
|
|
609
699
|
}
|
|
700
|
+
queryByTimeRange(query) {
|
|
701
|
+
return this.#objectMetaIndex.queryByTimeRange(query);
|
|
702
|
+
}
|
|
610
703
|
queryRelations(query) {
|
|
611
704
|
return this.#objectMetaIndex.queryRelations(query);
|
|
612
705
|
}
|
|
@@ -616,25 +709,26 @@ var IndexEngine = class {
|
|
|
616
709
|
lookupByObjectId(query) {
|
|
617
710
|
return this.#objectMetaIndex.lookupByObjectId(query);
|
|
618
711
|
}
|
|
619
|
-
update(dataSource, opts) {
|
|
712
|
+
update(ctx, dataSource, opts) {
|
|
620
713
|
return Effect5.gen(this, function* () {
|
|
621
|
-
|
|
622
|
-
const { updated: updatedFtsIndex, done: doneFtsIndex } = yield* this.#update(this.#ftsIndex, dataSource, {
|
|
623
|
-
indexName: "
|
|
714
|
+
const result = makeEmptyIndexingResult();
|
|
715
|
+
const { updated: updatedFtsIndex, done: doneFtsIndex, objects: ftsObjects } = yield* this.#update(ctx, this.#ftsIndex, dataSource, {
|
|
716
|
+
indexName: "fts5",
|
|
624
717
|
spaceId: opts.spaceId,
|
|
625
718
|
limit: opts.limit
|
|
626
719
|
});
|
|
627
|
-
updated += updatedFtsIndex;
|
|
628
|
-
|
|
720
|
+
result.updated += updatedFtsIndex;
|
|
721
|
+
result.done = result.done && doneFtsIndex;
|
|
722
|
+
accumulateIndexingResult(result, ftsObjects);
|
|
723
|
+
const { updated: updatedReverseRefIndex, done: doneReverseRefIndex, objects: reverseRefObjects } = yield* this.#update(ctx, this.#reverseRefIndex, dataSource, {
|
|
629
724
|
indexName: "reverseRef",
|
|
630
725
|
spaceId: opts.spaceId,
|
|
631
726
|
limit: opts.limit
|
|
632
727
|
});
|
|
633
|
-
updated += updatedReverseRefIndex;
|
|
634
|
-
|
|
635
|
-
|
|
636
|
-
|
|
637
|
-
};
|
|
728
|
+
result.updated += updatedReverseRefIndex;
|
|
729
|
+
result.done = result.done && doneReverseRefIndex;
|
|
730
|
+
accumulateIndexingResult(result, reverseRefObjects);
|
|
731
|
+
return result;
|
|
638
732
|
}).pipe(Effect5.withSpan("IndexEngine.update"));
|
|
639
733
|
}
|
|
640
734
|
/**
|
|
@@ -646,7 +740,7 @@ var IndexEngine = class {
|
|
|
646
740
|
* 4. Enriches objects with recordIds.
|
|
647
741
|
* 5. Updates the dependent index.
|
|
648
742
|
*/
|
|
649
|
-
#update(index, source, opts) {
|
|
743
|
+
#update(ctx, index, source, opts) {
|
|
650
744
|
return Effect5.gen(this, function* () {
|
|
651
745
|
const sqlTransaction = yield* SqlTransaction.SqlTransaction;
|
|
652
746
|
return yield* sqlTransaction.withTransaction(Effect5.gen(this, function* () {
|
|
@@ -656,13 +750,14 @@ var IndexEngine = class {
|
|
|
656
750
|
// Pass undefined to get all cursors when spaceId is null.
|
|
657
751
|
spaceId: opts.spaceId ?? void 0
|
|
658
752
|
});
|
|
659
|
-
const { objects, cursors: updatedCursors } = yield* source.getChangedObjects(cursors, {
|
|
753
|
+
const { objects, cursors: updatedCursors } = yield* source.getChangedObjects(ctx, cursors, {
|
|
660
754
|
limit: opts.limit
|
|
661
755
|
});
|
|
662
756
|
if (objects.length === 0) {
|
|
663
757
|
return {
|
|
664
758
|
updated: 0,
|
|
665
|
-
done: true
|
|
759
|
+
done: true,
|
|
760
|
+
objects: []
|
|
666
761
|
};
|
|
667
762
|
}
|
|
668
763
|
yield* this.#objectMetaIndex.update(objects);
|
|
@@ -677,7 +772,8 @@ var IndexEngine = class {
|
|
|
677
772
|
})));
|
|
678
773
|
return {
|
|
679
774
|
updated: objects.length,
|
|
680
|
-
done: false
|
|
775
|
+
done: false,
|
|
776
|
+
objects
|
|
681
777
|
};
|
|
682
778
|
}));
|
|
683
779
|
}).pipe(Effect5.withSpan("IndexEngine.#update"));
|