@dxos/index-core 0.8.4-main.59c2e9b → 0.8.4-main.60689f5b1c
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +102 -5
- package/README.md +1 -1
- package/dist/lib/{node-esm → neutral}/index.mjs +289 -60
- package/dist/lib/neutral/index.mjs.map +7 -0
- package/dist/lib/neutral/meta.json +1 -0
- package/dist/types/src/index-engine.d.ts +58 -9
- package/dist/types/src/index-engine.d.ts.map +1 -1
- package/dist/types/src/index-tracker.d.ts +3 -3
- package/dist/types/src/index-tracker.d.ts.map +1 -1
- package/dist/types/src/index.d.ts +2 -1
- package/dist/types/src/index.d.ts.map +1 -1
- package/dist/types/src/indexes/fts-index.d.ts +2 -1
- package/dist/types/src/indexes/fts-index.d.ts.map +1 -1
- package/dist/types/src/indexes/interface.d.ts +9 -0
- package/dist/types/src/indexes/interface.d.ts.map +1 -1
- package/dist/types/src/indexes/object-meta-index.d.ts +59 -2
- package/dist/types/src/indexes/object-meta-index.d.ts.map +1 -1
- package/dist/types/src/indexes/reverse-ref-index.d.ts +2 -2
- package/dist/types/src/indexes/reverse-ref-index.d.ts.map +1 -1
- package/dist/types/src/utils.d.ts +1 -1
- package/dist/types/src/utils.d.ts.map +1 -1
- package/dist/types/tsconfig.tsbuildinfo +1 -1
- package/package.json +19 -20
- package/src/index-engine.test.ts +168 -8
- package/src/index-engine.ts +150 -29
- package/src/index-tracker.ts +9 -0
- package/src/index.ts +8 -1
- package/src/indexes/fts-index.test.ts +153 -3
- package/src/indexes/fts-index.ts +25 -8
- package/src/indexes/interface.ts +10 -0
- package/src/indexes/object-meta-index.test.ts +366 -8
- package/src/indexes/object-meta-index.ts +310 -23
- package/src/indexes/reverse-ref-index.test.ts +31 -17
- package/src/indexes/reverse-ref-index.ts +11 -12
- package/src/utils.ts +1 -1
- package/dist/lib/browser/index.mjs +0 -559
- package/dist/lib/browser/index.mjs.map +0 -7
- package/dist/lib/browser/meta.json +0 -1
- package/dist/lib/node-esm/index.mjs.map +0 -7
- package/dist/lib/node-esm/meta.json +0 -1
package/LICENSE
CHANGED
|
@@ -1,8 +1,105 @@
|
|
|
1
|
-
|
|
2
|
-
Copyright (c) 2022 DXOS
|
|
1
|
+
# Functional Source License, Version 1.1, ALv2 Future License
|
|
3
2
|
|
|
4
|
-
|
|
3
|
+
## Abbreviation
|
|
5
4
|
|
|
6
|
-
|
|
5
|
+
FSL-1.1-Apache-2.0
|
|
7
6
|
|
|
8
|
-
|
|
7
|
+
## Notice
|
|
8
|
+
|
|
9
|
+
Copyright 2026 DXOS
|
|
10
|
+
|
|
11
|
+
## Terms and Conditions
|
|
12
|
+
|
|
13
|
+
### Licensor ("We")
|
|
14
|
+
|
|
15
|
+
The party offering the Software under these Terms and Conditions.
|
|
16
|
+
|
|
17
|
+
### The Software
|
|
18
|
+
|
|
19
|
+
The "Software" is each version of the software that we make available under
|
|
20
|
+
these Terms and Conditions, as indicated by our inclusion of these Terms and
|
|
21
|
+
Conditions with the Software.
|
|
22
|
+
|
|
23
|
+
### License Grant
|
|
24
|
+
|
|
25
|
+
Subject to your compliance with this License Grant and the Patents,
|
|
26
|
+
Redistribution and Trademark clauses below, we hereby grant you the right to
|
|
27
|
+
use, copy, modify, create derivative works, publicly perform, publicly display
|
|
28
|
+
and redistribute the Software for any Permitted Purpose identified below.
|
|
29
|
+
|
|
30
|
+
### Permitted Purpose
|
|
31
|
+
|
|
32
|
+
A Permitted Purpose is any purpose other than a Competing Use. A Competing Use
|
|
33
|
+
means making the Software available to others in a commercial product or
|
|
34
|
+
service that:
|
|
35
|
+
|
|
36
|
+
1. substitutes for the Software;
|
|
37
|
+
|
|
38
|
+
2. substitutes for any other product or service we offer using the Software
|
|
39
|
+
that exists as of the date we make the Software available; or
|
|
40
|
+
|
|
41
|
+
3. offers the same or substantially similar functionality as the Software.
|
|
42
|
+
|
|
43
|
+
Permitted Purposes specifically include using the Software:
|
|
44
|
+
|
|
45
|
+
1. for your internal use and access;
|
|
46
|
+
|
|
47
|
+
2. for non-commercial education;
|
|
48
|
+
|
|
49
|
+
3. for non-commercial research; and
|
|
50
|
+
|
|
51
|
+
4. in connection with professional services that you provide to a licensee
|
|
52
|
+
using the Software in accordance with these Terms and Conditions.
|
|
53
|
+
|
|
54
|
+
### Patents
|
|
55
|
+
|
|
56
|
+
To the extent your use for a Permitted Purpose would necessarily infringe our
|
|
57
|
+
patents, the license grant above includes a license under our patents. If you
|
|
58
|
+
make a claim against any party that the Software infringes or contributes to
|
|
59
|
+
the infringement of any patent, then your patent license to the Software ends
|
|
60
|
+
immediately.
|
|
61
|
+
|
|
62
|
+
### Redistribution
|
|
63
|
+
|
|
64
|
+
The Terms and Conditions apply to all copies, modifications and derivatives of
|
|
65
|
+
the Software.
|
|
66
|
+
|
|
67
|
+
If you redistribute any copies, modifications or derivatives of the Software,
|
|
68
|
+
you must include a copy of or a link to these Terms and Conditions and not
|
|
69
|
+
remove any copyright notices provided in or with the Software.
|
|
70
|
+
|
|
71
|
+
### Disclaimer
|
|
72
|
+
|
|
73
|
+
THE SOFTWARE IS PROVIDED "AS IS" AND WITHOUT WARRANTIES OF ANY KIND, EXPRESS OR
|
|
74
|
+
IMPLIED, INCLUDING WITHOUT LIMITATION WARRANTIES OF FITNESS FOR A PARTICULAR
|
|
75
|
+
PURPOSE, MERCHANTABILITY, TITLE OR NON-INFRINGEMENT.
|
|
76
|
+
|
|
77
|
+
IN NO EVENT WILL WE HAVE ANY LIABILITY TO YOU ARISING OUT OF OR RELATED TO THE
|
|
78
|
+
SOFTWARE, INCLUDING INDIRECT, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES,
|
|
79
|
+
EVEN IF WE HAVE BEEN INFORMED OF THEIR POSSIBILITY IN ADVANCE.
|
|
80
|
+
|
|
81
|
+
### Trademarks
|
|
82
|
+
|
|
83
|
+
Except for displaying the License Details and identifying us as the origin of
|
|
84
|
+
the Software, you have no right under these Terms and Conditions to use our
|
|
85
|
+
trademarks, trade names, service marks or product names.
|
|
86
|
+
|
|
87
|
+
## Grant of Future License
|
|
88
|
+
|
|
89
|
+
We hereby irrevocably grant you an additional license to use the Software under
|
|
90
|
+
the Apache License, Version 2.0 that is effective on the second anniversary of
|
|
91
|
+
the date we make the Software available. On or after that date, you may use the
|
|
92
|
+
Software under the Apache License, Version 2.0, in which case the following
|
|
93
|
+
will apply:
|
|
94
|
+
|
|
95
|
+
Licensed under the Apache License, Version 2.0 (the "License"); you may not use
|
|
96
|
+
this file except in compliance with the License.
|
|
97
|
+
|
|
98
|
+
You may obtain a copy of the License at
|
|
99
|
+
|
|
100
|
+
http://www.apache.org/licenses/LICENSE-2.0
|
|
101
|
+
|
|
102
|
+
Unless required by applicable law or agreed to in writing, software distributed
|
|
103
|
+
under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
|
|
104
|
+
CONDITIONS OF ANY KIND, either express or implied. See the License for the
|
|
105
|
+
specific language governing permissions and limitations under the License.
|
package/README.md
CHANGED
|
@@ -18,4 +18,4 @@ pnpm i @dxos/index-core
|
|
|
18
18
|
|
|
19
19
|
Your ideas, issues, and code are most welcome. Please take a look at our [community code of conduct](https://github.com/dxos/dxos/blob/main/CODE_OF_CONDUCT.md), the [issue guide](https://github.com/dxos/dxos/blob/main/CONTRIBUTING.md#submitting-issues), and the [PR contribution guide](https://github.com/dxos/dxos/blob/main/CONTRIBUTING.md#submitting-prs).
|
|
20
20
|
|
|
21
|
-
License: [
|
|
21
|
+
License: [FSL-1.1-Apache-2.0](./LICENSE) Copyright 2022 © DXOS
|
|
@@ -1,7 +1,6 @@
|
|
|
1
|
-
import { createRequire } from 'node:module';const require = createRequire(import.meta.url);
|
|
2
|
-
|
|
3
1
|
// src/index-engine.ts
|
|
4
2
|
import * as Effect5 from "effect/Effect";
|
|
3
|
+
import { ATTR_TYPE as ATTR_TYPE2 } from "@dxos/echo/internal";
|
|
5
4
|
import * as SqlTransaction from "@dxos/sql-sqlite/SqlTransaction";
|
|
6
5
|
|
|
7
6
|
// src/index-tracker.ts
|
|
@@ -33,6 +32,9 @@ var IndexCursor = Schema.Struct({
|
|
|
33
32
|
*/
|
|
34
33
|
cursor: Schema.Union(Schema.Number, Schema.String)
|
|
35
34
|
});
|
|
35
|
+
var DEPRECATED_INDEX_NAMES = [
|
|
36
|
+
"fts"
|
|
37
|
+
];
|
|
36
38
|
var IndexTracker = class {
|
|
37
39
|
migrate = Effect.fn("IndexTracker.migrate")(function* () {
|
|
38
40
|
const sql = yield* SqlClient.SqlClient;
|
|
@@ -44,6 +46,9 @@ var IndexTracker = class {
|
|
|
44
46
|
cursor,
|
|
45
47
|
PRIMARY KEY (indexName, spaceId, sourceName, resourceId)
|
|
46
48
|
)`;
|
|
49
|
+
yield* Effect.forEach(DEPRECATED_INDEX_NAMES, (indexName) => {
|
|
50
|
+
return sql`DELETE FROM indexCursor WHERE indexName = ${indexName}`;
|
|
51
|
+
});
|
|
47
52
|
});
|
|
48
53
|
queryCursors = Effect.fn("IndexTracker.queryCursors")((query) => Effect.gen(function* () {
|
|
49
54
|
const sql = yield* SqlClient.SqlClient;
|
|
@@ -84,6 +89,7 @@ var IndexTracker = class {
|
|
|
84
89
|
// src/indexes/fts-index.ts
|
|
85
90
|
import * as SqlClient3 from "@effect/sql/SqlClient";
|
|
86
91
|
import * as Effect2 from "effect/Effect";
|
|
92
|
+
var SQL_CHUNK_SIZE = 500;
|
|
87
93
|
var escapeFts5Query = (text) => {
|
|
88
94
|
return text.split(/\s+/).filter(Boolean).map((term) => `"${term.replace(/"/g, '""')}"`).join(" ");
|
|
89
95
|
};
|
|
@@ -145,6 +151,7 @@ var FtsIndex = class {
|
|
|
145
151
|
/**
|
|
146
152
|
* Query snapshots by recordIds.
|
|
147
153
|
* Returns the parsed JSON snapshots for queue objects.
|
|
154
|
+
* RecordIds not present in the FTS index are silently omitted from the result.
|
|
148
155
|
*/
|
|
149
156
|
querySnapshotsJSON(recordIds) {
|
|
150
157
|
return Effect2.gen(function* () {
|
|
@@ -152,11 +159,21 @@ var FtsIndex = class {
|
|
|
152
159
|
return [];
|
|
153
160
|
}
|
|
154
161
|
const sql = yield* SqlClient3.SqlClient;
|
|
155
|
-
const
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
162
|
+
const chunks = [];
|
|
163
|
+
for (let i = 0; i < recordIds.length; i += SQL_CHUNK_SIZE) {
|
|
164
|
+
chunks.push(recordIds.slice(i, i + SQL_CHUNK_SIZE));
|
|
165
|
+
}
|
|
166
|
+
const allResults = [];
|
|
167
|
+
for (const chunk of chunks) {
|
|
168
|
+
const rows = yield* sql`SELECT rowid, snapshot FROM ftsIndex WHERE rowid IN ${sql.in(chunk)}`;
|
|
169
|
+
for (const r of rows) {
|
|
170
|
+
allResults.push({
|
|
171
|
+
recordId: r.rowid,
|
|
172
|
+
snapshot: JSON.parse(r.snapshot)
|
|
173
|
+
});
|
|
174
|
+
}
|
|
175
|
+
}
|
|
176
|
+
return allResults;
|
|
160
177
|
});
|
|
161
178
|
}
|
|
162
179
|
update = Effect2.fn("FtsIndex.update")((objects) => Effect2.gen(function* () {
|
|
@@ -182,21 +199,52 @@ var FtsIndex = class {
|
|
|
182
199
|
import * as SqlClient5 from "@effect/sql/SqlClient";
|
|
183
200
|
import * as Effect3 from "effect/Effect";
|
|
184
201
|
import * as Schema2 from "effect/Schema";
|
|
185
|
-
import { ATTR_DELETED, ATTR_RELATION_SOURCE, ATTR_RELATION_TARGET, ATTR_TYPE } from "@dxos/echo/internal";
|
|
202
|
+
import { ATTR_DELETED, ATTR_PARENT, ATTR_RELATION_SOURCE, ATTR_RELATION_TARGET, ATTR_TYPE } from "@dxos/echo/internal";
|
|
203
|
+
import { DXN } from "@dxos/keys";
|
|
204
|
+
var _escapeLikePrefix = (prefix) => {
|
|
205
|
+
const escaped = prefix.replaceAll("\\", "\\\\").replaceAll("%", "\\%").replaceAll("_", "\\_");
|
|
206
|
+
return `${escaped}:%`;
|
|
207
|
+
};
|
|
186
208
|
var ObjectMeta = Schema2.Struct({
|
|
187
209
|
recordId: Schema2.Number,
|
|
188
210
|
objectId: Schema2.String,
|
|
189
211
|
queueId: Schema2.String,
|
|
212
|
+
/** Queue subspace namespace (e.g. 'data', 'trace'). Empty string for non-queue objects. */
|
|
213
|
+
queueNamespace: Schema2.String,
|
|
190
214
|
spaceId: Schema2.String,
|
|
191
215
|
documentId: Schema2.String,
|
|
192
216
|
entityKind: Schema2.String,
|
|
193
|
-
|
|
217
|
+
/** The versioned DXN of the type of the object. */
|
|
218
|
+
typeDXN: Schema2.String,
|
|
194
219
|
deleted: Schema2.Boolean,
|
|
195
220
|
source: Schema2.NullOr(Schema2.String),
|
|
196
221
|
target: Schema2.NullOr(Schema2.String),
|
|
222
|
+
/** Parent object id (nullable). */
|
|
223
|
+
parent: Schema2.NullOr(Schema2.String),
|
|
197
224
|
/** Monotonically increasing sequence number assigned on insert/update for tracking indexing order. */
|
|
198
|
-
version: Schema2.Number
|
|
225
|
+
version: Schema2.Number,
|
|
226
|
+
/** Unix ms timestamp when the object was first indexed. */
|
|
227
|
+
createdAt: Schema2.NullOr(Schema2.Number),
|
|
228
|
+
/** Unix ms timestamp when the object was last re-indexed. */
|
|
229
|
+
updatedAt: Schema2.NullOr(Schema2.Number)
|
|
199
230
|
});
|
|
231
|
+
var buildSourceCondition = (sql, spaceIds, includeAllQueues, queueIds) => {
|
|
232
|
+
const conditions = [];
|
|
233
|
+
if (spaceIds.length > 0) {
|
|
234
|
+
if (includeAllQueues) {
|
|
235
|
+
conditions.push(sql`${sql.in("spaceId", spaceIds)}`);
|
|
236
|
+
} else {
|
|
237
|
+
conditions.push(sql`(${sql.in("spaceId", spaceIds)} AND queueId = '')`);
|
|
238
|
+
}
|
|
239
|
+
}
|
|
240
|
+
if (queueIds && queueIds.length > 0) {
|
|
241
|
+
conditions.push(sql`${sql.in("queueId", queueIds)}`);
|
|
242
|
+
}
|
|
243
|
+
if (conditions.length === 0) {
|
|
244
|
+
return sql`1 = 0`;
|
|
245
|
+
}
|
|
246
|
+
return sql.or(conditions);
|
|
247
|
+
};
|
|
200
248
|
var ObjectMetaIndex = class {
|
|
201
249
|
migrate = Effect3.fn("ObjectMetaIndex.runMigrations")(function* () {
|
|
202
250
|
const sql = yield* SqlClient5.SqlClient;
|
|
@@ -204,22 +252,89 @@ var ObjectMetaIndex = class {
|
|
|
204
252
|
recordId INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
205
253
|
objectId TEXT NOT NULL,
|
|
206
254
|
queueId TEXT NOT NULL DEFAULT '',
|
|
255
|
+
queueNamespace TEXT NOT NULL DEFAULT '',
|
|
207
256
|
spaceId TEXT NOT NULL,
|
|
208
257
|
documentId TEXT NOT NULL DEFAULT '',
|
|
209
258
|
entityKind TEXT NOT NULL,
|
|
210
|
-
|
|
259
|
+
typeDXN TEXT NOT NULL,
|
|
211
260
|
deleted INTEGER NOT NULL,
|
|
212
261
|
source TEXT,
|
|
213
262
|
target TEXT,
|
|
214
|
-
|
|
263
|
+
parent TEXT,
|
|
264
|
+
version INTEGER NOT NULL,
|
|
265
|
+
createdAt INTEGER,
|
|
266
|
+
updatedAt INTEGER
|
|
215
267
|
)`;
|
|
268
|
+
yield* Effect3.catchAll(sql`ALTER TABLE objectMeta ADD COLUMN parent TEXT`, () => Effect3.void);
|
|
269
|
+
yield* Effect3.catchAll(sql`ALTER TABLE objectMeta ADD COLUMN createdAt INTEGER`, () => Effect3.void);
|
|
270
|
+
yield* Effect3.catchAll(sql`ALTER TABLE objectMeta ADD COLUMN updatedAt INTEGER`, () => Effect3.void);
|
|
271
|
+
yield* Effect3.catchAll(sql`ALTER TABLE objectMeta ADD COLUMN queueNamespace TEXT NOT NULL DEFAULT ''`, () => Effect3.void);
|
|
216
272
|
yield* sql`CREATE INDEX IF NOT EXISTS idx_object_index_objectId ON objectMeta(spaceId, objectId)`;
|
|
217
|
-
yield* sql`CREATE INDEX IF NOT EXISTS
|
|
273
|
+
yield* sql`CREATE INDEX IF NOT EXISTS idx_object_index_typeDXN ON objectMeta(spaceId, typeDXN)`;
|
|
218
274
|
yield* sql`CREATE INDEX IF NOT EXISTS idx_object_index_version ON objectMeta(version)`;
|
|
275
|
+
yield* sql`CREATE INDEX IF NOT EXISTS idx_object_index_parent ON objectMeta(spaceId, parent)`;
|
|
276
|
+
yield* sql`CREATE INDEX IF NOT EXISTS idx_object_index_updatedAt ON objectMeta(updatedAt)`;
|
|
277
|
+
yield* sql`CREATE INDEX IF NOT EXISTS idx_object_index_createdAt ON objectMeta(createdAt)`;
|
|
219
278
|
});
|
|
220
|
-
query = Effect3.fn("ObjectMetaIndex.
|
|
279
|
+
query = Effect3.fn("ObjectMetaIndex.query")((query) => Effect3.gen(function* () {
|
|
280
|
+
const sql = yield* SqlClient5.SqlClient;
|
|
281
|
+
const parsedType = DXN.tryParse(query.typeDXN)?.asTypeDXN();
|
|
282
|
+
const rows = parsedType && parsedType.version === void 0 ? yield* sql`SELECT * FROM objectMeta WHERE spaceId = ${query.spaceId} AND (typeDXN = ${query.typeDXN} OR typeDXN LIKE ${_escapeLikePrefix(query.typeDXN)} ESCAPE '\\')` : yield* sql`SELECT * FROM objectMeta WHERE spaceId = ${query.spaceId} AND typeDXN = ${query.typeDXN}`;
|
|
283
|
+
return rows.map((row) => ({
|
|
284
|
+
...row,
|
|
285
|
+
deleted: !!row.deleted
|
|
286
|
+
}));
|
|
287
|
+
}));
|
|
288
|
+
queryAll = Effect3.fn("ObjectMetaIndex.queryAll")((query) => Effect3.gen(function* () {
|
|
289
|
+
if (query.spaceIds.length === 0 && (!query.queueIds || query.queueIds.length === 0)) {
|
|
290
|
+
return [];
|
|
291
|
+
}
|
|
292
|
+
const sql = yield* SqlClient5.SqlClient;
|
|
293
|
+
const sourceCondition = buildSourceCondition(sql, query.spaceIds, query.includeAllQueues ?? false, query.queueIds ?? null);
|
|
294
|
+
const rows = yield* sql`SELECT * FROM objectMeta WHERE ${sourceCondition}`;
|
|
295
|
+
return rows.map((row) => ({
|
|
296
|
+
...row,
|
|
297
|
+
deleted: !!row.deleted
|
|
298
|
+
}));
|
|
299
|
+
}));
|
|
300
|
+
queryTypes = Effect3.fn("ObjectMetaIndex.queryTypes")(({ spaceIds, typeDxns, inverted = false, includeAllQueues = false, queueIds = null }) => Effect3.gen(function* () {
|
|
301
|
+
if (spaceIds.length === 0 && (!queueIds || queueIds.length === 0)) {
|
|
302
|
+
return [];
|
|
303
|
+
}
|
|
304
|
+
if (typeDxns.length === 0) {
|
|
305
|
+
if (!inverted) {
|
|
306
|
+
return [];
|
|
307
|
+
}
|
|
308
|
+
const sql2 = yield* SqlClient5.SqlClient;
|
|
309
|
+
const sourceCondition2 = buildSourceCondition(sql2, spaceIds, includeAllQueues, queueIds);
|
|
310
|
+
const rows2 = yield* sql2`SELECT * FROM objectMeta WHERE ${sourceCondition2}`;
|
|
311
|
+
return rows2.map((row) => ({
|
|
312
|
+
...row,
|
|
313
|
+
deleted: !!row.deleted
|
|
314
|
+
}));
|
|
315
|
+
}
|
|
316
|
+
const sql = yield* SqlClient5.SqlClient;
|
|
317
|
+
const sourceCondition = buildSourceCondition(sql, spaceIds, includeAllQueues, queueIds);
|
|
318
|
+
const typeWhere = sql.or(typeDxns.map((typeDXN) => {
|
|
319
|
+
const parsedType = DXN.tryParse(typeDXN)?.asTypeDXN();
|
|
320
|
+
return parsedType && parsedType.version === void 0 ? sql.or([
|
|
321
|
+
sql`typeDXN = ${typeDXN}`,
|
|
322
|
+
sql`typeDXN LIKE ${_escapeLikePrefix(typeDXN)} ESCAPE '\\'`
|
|
323
|
+
]) : sql`typeDXN = ${typeDXN}`;
|
|
324
|
+
}));
|
|
325
|
+
const rows = inverted ? yield* sql`SELECT * FROM objectMeta WHERE ${sourceCondition} AND NOT ${typeWhere}` : yield* sql`SELECT * FROM objectMeta WHERE ${sourceCondition} AND ${typeWhere}`;
|
|
326
|
+
return rows.map((row) => ({
|
|
327
|
+
...row,
|
|
328
|
+
deleted: !!row.deleted
|
|
329
|
+
}));
|
|
330
|
+
}));
|
|
331
|
+
queryRelations = Effect3.fn("ObjectMetaIndex.queryRelations")(({ endpoint, anchorDxns }) => Effect3.gen(function* () {
|
|
332
|
+
if (anchorDxns.length === 0) {
|
|
333
|
+
return [];
|
|
334
|
+
}
|
|
221
335
|
const sql = yield* SqlClient5.SqlClient;
|
|
222
|
-
const
|
|
336
|
+
const column = endpoint === "source" ? "source" : "target";
|
|
337
|
+
const rows = yield* sql`SELECT * FROM objectMeta WHERE entityKind = 'relation' AND ${sql.in(column, anchorDxns)}`;
|
|
223
338
|
return rows.map((row) => ({
|
|
224
339
|
...row,
|
|
225
340
|
deleted: !!row.deleted
|
|
@@ -229,7 +344,7 @@ var ObjectMetaIndex = class {
|
|
|
229
344
|
update = Effect3.fn("ObjectMetaIndex.update")((objects) => Effect3.gen(function* () {
|
|
230
345
|
const sql = yield* SqlClient5.SqlClient;
|
|
231
346
|
yield* Effect3.forEach(objects, (object) => Effect3.gen(function* () {
|
|
232
|
-
const { spaceId, queueId, documentId, data } = object;
|
|
347
|
+
const { spaceId, queueId, queueNamespace, documentId, data } = object;
|
|
233
348
|
const castData = data;
|
|
234
349
|
const objectId = castData.id;
|
|
235
350
|
let existing;
|
|
@@ -244,30 +359,37 @@ var ObjectMetaIndex = class {
|
|
|
244
359
|
const [{ v }] = result;
|
|
245
360
|
const version = (v ?? 0) + 1;
|
|
246
361
|
const entityKind = castData[ATTR_RELATION_SOURCE] ? "relation" : "object";
|
|
247
|
-
const
|
|
362
|
+
const typeDXN = castData[ATTR_TYPE] ? String(castData[ATTR_TYPE]) : "type";
|
|
248
363
|
const deleted = castData[ATTR_DELETED] ? 1 : 0;
|
|
249
364
|
const source = entityKind === "relation" ? castData[ATTR_RELATION_SOURCE] ?? null : null;
|
|
250
365
|
const target = entityKind === "relation" ? castData[ATTR_RELATION_TARGET] ?? null : null;
|
|
366
|
+
const parent = castData[ATTR_PARENT] ?? null;
|
|
367
|
+
const sourceTimestamp = object.updatedAt;
|
|
251
368
|
if (existing.length > 0) {
|
|
252
369
|
yield* sql`
|
|
253
370
|
UPDATE objectMeta SET
|
|
254
371
|
version = ${version},
|
|
372
|
+
queueNamespace = ${queueNamespace ?? ""},
|
|
255
373
|
entityKind = ${entityKind},
|
|
256
|
-
|
|
374
|
+
typeDXN = ${typeDXN},
|
|
257
375
|
deleted = ${deleted},
|
|
258
376
|
source = ${source},
|
|
259
|
-
target = ${target}
|
|
377
|
+
target = ${target},
|
|
378
|
+
parent = ${parent},
|
|
379
|
+
updatedAt = ${sourceTimestamp}
|
|
260
380
|
WHERE recordId = ${existing[0].recordId}
|
|
261
381
|
`;
|
|
262
382
|
} else {
|
|
263
383
|
yield* sql`
|
|
264
384
|
INSERT INTO objectMeta (
|
|
265
|
-
objectId, queueId, spaceId, documentId,
|
|
266
|
-
entityKind,
|
|
385
|
+
objectId, queueId, queueNamespace, spaceId, documentId,
|
|
386
|
+
entityKind, typeDXN, deleted, source, target, parent, version,
|
|
387
|
+
createdAt, updatedAt
|
|
267
388
|
) VALUES (
|
|
268
|
-
${objectId}, ${queueId ?? ""}, ${spaceId}, ${documentId ?? ""},
|
|
269
|
-
${entityKind}, ${
|
|
270
|
-
${source}, ${target}, ${version}
|
|
389
|
+
${objectId}, ${queueId ?? ""}, ${queueNamespace ?? ""}, ${spaceId}, ${documentId ?? ""},
|
|
390
|
+
${entityKind}, ${typeDXN}, ${deleted},
|
|
391
|
+
${source}, ${target}, ${parent}, ${version},
|
|
392
|
+
${sourceTimestamp}, ${sourceTimestamp}
|
|
271
393
|
)
|
|
272
394
|
`;
|
|
273
395
|
}
|
|
@@ -293,7 +415,7 @@ var ObjectMetaIndex = class {
|
|
|
293
415
|
result = [];
|
|
294
416
|
}
|
|
295
417
|
if (result.length === 0) {
|
|
296
|
-
yield* Effect3.die(new Error(`Object not found in ObjectMetaIndex: ${spaceId}/${documentId ?? queueId}/${objectId}`));
|
|
418
|
+
return yield* Effect3.die(new Error(`Object not found in ObjectMetaIndex: ${spaceId}/${documentId ?? queueId}/${objectId}`));
|
|
297
419
|
}
|
|
298
420
|
object.recordId = result[0].recordId;
|
|
299
421
|
}
|
|
@@ -306,8 +428,68 @@ var ObjectMetaIndex = class {
|
|
|
306
428
|
return [];
|
|
307
429
|
}
|
|
308
430
|
const sql = yield* SqlClient5.SqlClient;
|
|
309
|
-
const
|
|
310
|
-
|
|
431
|
+
const rows = yield* sql`SELECT * FROM objectMeta WHERE ${sql.in("recordId", recordIds)}`;
|
|
432
|
+
return rows.map((row) => ({
|
|
433
|
+
...row,
|
|
434
|
+
deleted: !!row.deleted
|
|
435
|
+
}));
|
|
436
|
+
}));
|
|
437
|
+
/**
|
|
438
|
+
* Look up object metadata by objectId, spaceId, and queueId.
|
|
439
|
+
*/
|
|
440
|
+
lookupByObjectId = Effect3.fn("ObjectMetaIndex.lookupByObjectId")((query) => Effect3.gen(function* () {
|
|
441
|
+
const sql = yield* SqlClient5.SqlClient;
|
|
442
|
+
const rows = yield* sql`SELECT * FROM objectMeta WHERE spaceId = ${query.spaceId} AND queueId = ${query.queueId} AND objectId = ${query.objectId} LIMIT 1`;
|
|
443
|
+
if (rows.length === 0) {
|
|
444
|
+
return null;
|
|
445
|
+
}
|
|
446
|
+
return {
|
|
447
|
+
...rows[0],
|
|
448
|
+
deleted: !!rows[0].deleted
|
|
449
|
+
};
|
|
450
|
+
}));
|
|
451
|
+
/**
|
|
452
|
+
* Query objects by timestamp range.
|
|
453
|
+
*/
|
|
454
|
+
queryByTimeRange = Effect3.fn("ObjectMetaIndex.queryByTimeRange")((query) => Effect3.gen(function* () {
|
|
455
|
+
if (query.spaceIds.length === 0 && (!query.queueIds || query.queueIds.length === 0)) {
|
|
456
|
+
return [];
|
|
457
|
+
}
|
|
458
|
+
const sql = yield* SqlClient5.SqlClient;
|
|
459
|
+
const sourceCondition = buildSourceCondition(sql, query.spaceIds, query.includeAllQueues ?? false, query.queueIds ?? null);
|
|
460
|
+
const timeConditions = [];
|
|
461
|
+
if (query.updatedAfter != null) {
|
|
462
|
+
timeConditions.push(sql`updatedAt >= ${query.updatedAfter}`);
|
|
463
|
+
}
|
|
464
|
+
if (query.updatedBefore != null) {
|
|
465
|
+
timeConditions.push(sql`updatedAt <= ${query.updatedBefore}`);
|
|
466
|
+
}
|
|
467
|
+
if (query.createdAfter != null) {
|
|
468
|
+
timeConditions.push(sql`createdAt >= ${query.createdAfter}`);
|
|
469
|
+
}
|
|
470
|
+
if (query.createdBefore != null) {
|
|
471
|
+
timeConditions.push(sql`createdAt <= ${query.createdBefore}`);
|
|
472
|
+
}
|
|
473
|
+
const rows = timeConditions.length > 0 ? yield* sql`SELECT * FROM objectMeta WHERE ${sourceCondition} AND ${sql.and(timeConditions)}` : yield* sql`SELECT * FROM objectMeta WHERE ${sourceCondition}`;
|
|
474
|
+
return rows.map((row) => ({
|
|
475
|
+
...row,
|
|
476
|
+
deleted: !!row.deleted
|
|
477
|
+
}));
|
|
478
|
+
}));
|
|
479
|
+
/**
|
|
480
|
+
* Query children by parent object ids.
|
|
481
|
+
* Matches both:
|
|
482
|
+
* - Objects whose `parent` field references one of the given parent ids (standard parent/child hierarchy).
|
|
483
|
+
* - Queue items whose `queueId` equals one of the parent ids (e.g. items inside a Feed, since a feed's queue
|
|
484
|
+
* DXN uses the feed's object id as its queue id — see `Feed.getQueueDxn`).
|
|
485
|
+
*/
|
|
486
|
+
queryChildren = Effect3.fn("ObjectMetaIndex.queryChildren")((query) => Effect3.gen(function* () {
|
|
487
|
+
if (query.parentIds.length === 0) {
|
|
488
|
+
return [];
|
|
489
|
+
}
|
|
490
|
+
const sql = yield* SqlClient5.SqlClient;
|
|
491
|
+
const parentDxns = query.parentIds.map((id) => DXN.fromLocalObjectId(id).toString());
|
|
492
|
+
const rows = yield* sql`SELECT * FROM objectMeta WHERE ${sql.in("spaceId", query.spaceId)} AND (${sql.in("parent", parentDxns)} OR ${sql.in("queueId", query.parentIds)})`;
|
|
311
493
|
return rows.map((row) => ({
|
|
312
494
|
...row,
|
|
313
495
|
deleted: !!row.deleted
|
|
@@ -336,15 +518,7 @@ var EscapedPropPath = class extends Schema3.String.annotations({
|
|
|
336
518
|
let current = "";
|
|
337
519
|
for (let i = 0; i < path.length; i++) {
|
|
338
520
|
if (path[i] === "\\") {
|
|
339
|
-
invariant(i + 1 < path.length && (path[i + 1] === "." || path[i + 1] === "\\"), "Malformed escaping.", {
|
|
340
|
-
F: __dxlog_file,
|
|
341
|
-
L: 34,
|
|
342
|
-
S: this,
|
|
343
|
-
A: [
|
|
344
|
-
"i + 1 < path.length && (path[i + 1] === '.' || path[i + 1] === '\\\\')",
|
|
345
|
-
"'Malformed escaping.'"
|
|
346
|
-
]
|
|
347
|
-
});
|
|
521
|
+
invariant(i + 1 < path.length && (path[i + 1] === "." || path[i + 1] === "\\"), "Malformed escaping.", { "~LogMeta": "~LogMeta", F: __dxlog_file, L: 25, S: this, A: ["i + 1 < path.length && (path[i + 1] === '.' || path[i + 1] === '\\\\')", "'Malformed escaping.'"] });
|
|
348
522
|
current = current + path[i + 1];
|
|
349
523
|
i++;
|
|
350
524
|
} else if (path[i] === ".") {
|
|
@@ -371,7 +545,7 @@ var extractReferences = (data) => {
|
|
|
371
545
|
}
|
|
372
546
|
refs.push({
|
|
373
547
|
path,
|
|
374
|
-
|
|
548
|
+
targetDXN: dxn.toString()
|
|
375
549
|
});
|
|
376
550
|
} else if (typeof value === "object" && value !== null && !Array.isArray(value)) {
|
|
377
551
|
for (const [key, v] of Object.entries(value)) {
|
|
@@ -394,7 +568,7 @@ var extractReferences = (data) => {
|
|
|
394
568
|
};
|
|
395
569
|
var ReverseRef = Schema4.Struct({
|
|
396
570
|
recordId: Schema4.Number,
|
|
397
|
-
|
|
571
|
+
targetDXN: Schema4.String,
|
|
398
572
|
/**
|
|
399
573
|
* Escaped property path within an object.
|
|
400
574
|
*
|
|
@@ -411,18 +585,18 @@ var ReverseRefIndex = class {
|
|
|
411
585
|
const sql = yield* SqlClient7.SqlClient;
|
|
412
586
|
yield* sql`CREATE TABLE IF NOT EXISTS reverseRef (
|
|
413
587
|
recordId INTEGER NOT NULL,
|
|
414
|
-
|
|
588
|
+
targetDXN TEXT NOT NULL,
|
|
415
589
|
propPath TEXT NOT NULL,
|
|
416
|
-
PRIMARY KEY (recordId,
|
|
590
|
+
PRIMARY KEY (recordId, targetDXN, propPath)
|
|
417
591
|
)`;
|
|
418
|
-
yield* sql`CREATE INDEX IF NOT EXISTS idx_reverse_ref_target ON reverseRef(
|
|
592
|
+
yield* sql`CREATE INDEX IF NOT EXISTS idx_reverse_ref_target ON reverseRef(targetDXN)`;
|
|
419
593
|
});
|
|
420
594
|
/**
|
|
421
595
|
* Query all references pointing to a target DXN.
|
|
422
596
|
*/
|
|
423
|
-
query = Effect4.fn("ReverseRefIndex.query")(({
|
|
597
|
+
query = Effect4.fn("ReverseRefIndex.query")(({ targetDXN }) => Effect4.gen(function* () {
|
|
424
598
|
const sql = yield* SqlClient7.SqlClient;
|
|
425
|
-
const rows = yield* sql`SELECT * FROM reverseRef WHERE
|
|
599
|
+
const rows = yield* sql`SELECT * FROM reverseRef WHERE targetDXN = ${targetDXN}`;
|
|
426
600
|
return rows;
|
|
427
601
|
}));
|
|
428
602
|
update = Effect4.fn("ReverseRefIndex.update")((objects) => Effect4.gen(function* () {
|
|
@@ -434,7 +608,7 @@ var ReverseRefIndex = class {
|
|
|
434
608
|
}
|
|
435
609
|
yield* sql`DELETE FROM reverseRef WHERE recordId = ${recordId}`;
|
|
436
610
|
const refs = extractReferences(data);
|
|
437
|
-
yield* Effect4.forEach(refs, (ref) => sql`INSERT INTO reverseRef (recordId,
|
|
611
|
+
yield* Effect4.forEach(refs, (ref) => sql`INSERT INTO reverseRef (recordId, targetDXN, propPath) VALUES (${recordId}, ${ref.targetDXN}, ${EscapedPropPath.escape(ref.path)})`, {
|
|
438
612
|
discard: true
|
|
439
613
|
});
|
|
440
614
|
}), {
|
|
@@ -444,6 +618,33 @@ var ReverseRefIndex = class {
|
|
|
444
618
|
};
|
|
445
619
|
|
|
446
620
|
// src/index-engine.ts
|
|
621
|
+
var makeEmptyIndexingResult = () => ({
|
|
622
|
+
updated: 0,
|
|
623
|
+
done: true,
|
|
624
|
+
spaces: /* @__PURE__ */ new Set(),
|
|
625
|
+
queues: /* @__PURE__ */ new Set(),
|
|
626
|
+
documents: /* @__PURE__ */ new Set(),
|
|
627
|
+
types: /* @__PURE__ */ new Set(),
|
|
628
|
+
objects: /* @__PURE__ */ new Set()
|
|
629
|
+
});
|
|
630
|
+
var accumulateIndexingResult = (acc, objects) => {
|
|
631
|
+
for (const obj of objects) {
|
|
632
|
+
acc.spaces.add(obj.spaceId);
|
|
633
|
+
if (obj.queueId) {
|
|
634
|
+
acc.queues.add(obj.queueId);
|
|
635
|
+
}
|
|
636
|
+
if (obj.documentId) {
|
|
637
|
+
acc.documents.add(obj.documentId);
|
|
638
|
+
}
|
|
639
|
+
const t = obj.data[ATTR_TYPE2];
|
|
640
|
+
if (t) {
|
|
641
|
+
acc.types.add(String(t));
|
|
642
|
+
}
|
|
643
|
+
if (obj.data.id) {
|
|
644
|
+
acc.objects.add(obj.data.id);
|
|
645
|
+
}
|
|
646
|
+
}
|
|
647
|
+
};
|
|
447
648
|
var IndexEngine = class {
|
|
448
649
|
#tracker;
|
|
449
650
|
#objectMetaIndex;
|
|
@@ -474,6 +675,9 @@ var IndexEngine = class {
|
|
|
474
675
|
queryReverseRef(query) {
|
|
475
676
|
return this.#reverseRefIndex.query(query);
|
|
476
677
|
}
|
|
678
|
+
queryAll(query) {
|
|
679
|
+
return this.#objectMetaIndex.queryAll(query);
|
|
680
|
+
}
|
|
477
681
|
/**
|
|
478
682
|
* Query snapshots by recordIds.
|
|
479
683
|
* Used to load queue objects from indexed snapshots.
|
|
@@ -484,25 +688,47 @@ var IndexEngine = class {
|
|
|
484
688
|
queryType(query) {
|
|
485
689
|
return this.#objectMetaIndex.query(query);
|
|
486
690
|
}
|
|
487
|
-
|
|
691
|
+
/**
|
|
692
|
+
* Query children by parent object ids.
|
|
693
|
+
*/
|
|
694
|
+
queryChildren(query) {
|
|
695
|
+
return this.#objectMetaIndex.queryChildren(query);
|
|
696
|
+
}
|
|
697
|
+
queryTypes(query) {
|
|
698
|
+
return this.#objectMetaIndex.queryTypes(query);
|
|
699
|
+
}
|
|
700
|
+
queryByTimeRange(query) {
|
|
701
|
+
return this.#objectMetaIndex.queryByTimeRange(query);
|
|
702
|
+
}
|
|
703
|
+
queryRelations(query) {
|
|
704
|
+
return this.#objectMetaIndex.queryRelations(query);
|
|
705
|
+
}
|
|
706
|
+
lookupByRecordIds(recordIds) {
|
|
707
|
+
return this.#objectMetaIndex.lookupByRecordIds(recordIds);
|
|
708
|
+
}
|
|
709
|
+
lookupByObjectId(query) {
|
|
710
|
+
return this.#objectMetaIndex.lookupByObjectId(query);
|
|
711
|
+
}
|
|
712
|
+
update(ctx, dataSource, opts) {
|
|
488
713
|
return Effect5.gen(this, function* () {
|
|
489
|
-
|
|
490
|
-
const { updated: updatedFtsIndex, done: doneFtsIndex } = yield* this.#update(this.#ftsIndex, dataSource, {
|
|
491
|
-
indexName: "
|
|
714
|
+
const result = makeEmptyIndexingResult();
|
|
715
|
+
const { updated: updatedFtsIndex, done: doneFtsIndex, objects: ftsObjects } = yield* this.#update(ctx, this.#ftsIndex, dataSource, {
|
|
716
|
+
indexName: "fts5",
|
|
492
717
|
spaceId: opts.spaceId,
|
|
493
718
|
limit: opts.limit
|
|
494
719
|
});
|
|
495
|
-
updated += updatedFtsIndex;
|
|
496
|
-
|
|
720
|
+
result.updated += updatedFtsIndex;
|
|
721
|
+
result.done = result.done && doneFtsIndex;
|
|
722
|
+
accumulateIndexingResult(result, ftsObjects);
|
|
723
|
+
const { updated: updatedReverseRefIndex, done: doneReverseRefIndex, objects: reverseRefObjects } = yield* this.#update(ctx, this.#reverseRefIndex, dataSource, {
|
|
497
724
|
indexName: "reverseRef",
|
|
498
725
|
spaceId: opts.spaceId,
|
|
499
726
|
limit: opts.limit
|
|
500
727
|
});
|
|
501
|
-
updated += updatedReverseRefIndex;
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
};
|
|
728
|
+
result.updated += updatedReverseRefIndex;
|
|
729
|
+
result.done = result.done && doneReverseRefIndex;
|
|
730
|
+
accumulateIndexingResult(result, reverseRefObjects);
|
|
731
|
+
return result;
|
|
506
732
|
}).pipe(Effect5.withSpan("IndexEngine.update"));
|
|
507
733
|
}
|
|
508
734
|
/**
|
|
@@ -514,7 +740,7 @@ var IndexEngine = class {
|
|
|
514
740
|
* 4. Enriches objects with recordIds.
|
|
515
741
|
* 5. Updates the dependent index.
|
|
516
742
|
*/
|
|
517
|
-
#update(index, source, opts) {
|
|
743
|
+
#update(ctx, index, source, opts) {
|
|
518
744
|
return Effect5.gen(this, function* () {
|
|
519
745
|
const sqlTransaction = yield* SqlTransaction.SqlTransaction;
|
|
520
746
|
return yield* sqlTransaction.withTransaction(Effect5.gen(this, function* () {
|
|
@@ -524,13 +750,14 @@ var IndexEngine = class {
|
|
|
524
750
|
// Pass undefined to get all cursors when spaceId is null.
|
|
525
751
|
spaceId: opts.spaceId ?? void 0
|
|
526
752
|
});
|
|
527
|
-
const { objects, cursors: updatedCursors } = yield* source.getChangedObjects(cursors, {
|
|
753
|
+
const { objects, cursors: updatedCursors } = yield* source.getChangedObjects(ctx, cursors, {
|
|
528
754
|
limit: opts.limit
|
|
529
755
|
});
|
|
530
756
|
if (objects.length === 0) {
|
|
531
757
|
return {
|
|
532
758
|
updated: 0,
|
|
533
|
-
done: true
|
|
759
|
+
done: true,
|
|
760
|
+
objects: []
|
|
534
761
|
};
|
|
535
762
|
}
|
|
536
763
|
yield* this.#objectMetaIndex.update(objects);
|
|
@@ -545,13 +772,15 @@ var IndexEngine = class {
|
|
|
545
772
|
})));
|
|
546
773
|
return {
|
|
547
774
|
updated: objects.length,
|
|
548
|
-
done: false
|
|
775
|
+
done: false,
|
|
776
|
+
objects
|
|
549
777
|
};
|
|
550
778
|
}));
|
|
551
|
-
}).pipe(Effect5.withSpan("IndexEngine.#
|
|
779
|
+
}).pipe(Effect5.withSpan("IndexEngine.#update"));
|
|
552
780
|
}
|
|
553
781
|
};
|
|
554
782
|
export {
|
|
783
|
+
EscapedPropPath,
|
|
555
784
|
FtsIndex,
|
|
556
785
|
IndexEngine,
|
|
557
786
|
IndexTracker,
|