@lde/search-pipeline 0.10.0 → 0.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +46 -6
- package/dist/extraction.d.ts +43 -0
- package/dist/extraction.d.ts.map +1 -0
- package/dist/extraction.js +108 -0
- package/dist/index.d.ts +2 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -0
- package/dist/search-stages.d.ts +10 -2
- package/dist/search-stages.d.ts.map +1 -1
- package/dist/search-stages.js +12 -2
- package/package.json +6 -3
package/README.md
CHANGED
|
@@ -21,7 +21,12 @@ The division of labour ([ADR 6](../../docs/decisions/0006-make-the-writer-transa
|
|
|
21
21
|
**root-complete batch** into documents paired with their `SearchType`
|
|
22
22
|
(`TypedSearchDocument`) – so projection happens inside the batch and memory
|
|
23
23
|
is bounded by `batchSize` roots, not the dataset
|
|
24
|
-
([ADR 13](../../docs/decisions/0013-project-inside-the-batch-per-root-type.md))
|
|
24
|
+
([ADR 13](../../docs/decisions/0013-project-inside-the-batch-per-root-type.md)).
|
|
25
|
+
Extraction is **generated from the schema**: `extractionQuery` mints a
|
|
26
|
+
CONSTRUCT that reads each field’s source `path` (a SPARQL property path) and
|
|
27
|
+
emits it under the field’s IR Alias (`urn:lde:‹Type›/‹field›`) – the same key
|
|
28
|
+
the projection reads back – so the reader and the projection cannot drift. A
|
|
29
|
+
stage without an explicit `readers` defaults to this generated reader;
|
|
25
30
|
- `searchIndexWriter` is the pipeline’s single terminal: an engine-agnostic
|
|
26
31
|
router that dispatches each document to the engine writer for **its**
|
|
27
32
|
type’s collection ([ADR 9](../../docs/decisions/0009-route-a-whole-schema-projection-to-per-type-collections.md)).
|
|
@@ -32,7 +37,7 @@ So a search pipeline is **one terminal, N stages**: `new Pipeline<TypedSearchDoc
|
|
|
32
37
|
## Usage
|
|
33
38
|
|
|
34
39
|
```typescript
|
|
35
|
-
import { Pipeline
|
|
40
|
+
import { Pipeline } from '@lde/pipeline';
|
|
36
41
|
import { searchSchema } from '@lde/search';
|
|
37
42
|
import { BlueGreenRebuild } from '@lde/search-typesense';
|
|
38
43
|
import {
|
|
@@ -83,8 +88,11 @@ const pipeline = new Pipeline<TypedSearchDocument>({
|
|
|
83
88
|
// One stage per root type: each selects its own roots (here by class –
|
|
84
89
|
// `selectByClass` is a convenience for the object grain; a deployment writes
|
|
85
90
|
// its own selector where the entry point is a domain fact) and extracts each
|
|
86
|
-
// root’s quads. `
|
|
87
|
-
//
|
|
91
|
+
// root’s quads. Omitting `readers` defaults each stage to the Extraction
|
|
92
|
+
// CONSTRUCT generated from the schema (pass one only for a non-SPARQL source or
|
|
93
|
+
// to merge readers). `rootVariable` couples the selector’s projected variable
|
|
94
|
+
// to the CONSTRUCT’s free subject; it must not be `dataset` (reserved by the
|
|
95
|
+
// reader).
|
|
88
96
|
stages: searchStages({
|
|
89
97
|
schema,
|
|
90
98
|
types: [
|
|
@@ -92,13 +100,11 @@ const pipeline = new Pipeline<TypedSearchDocument>({
|
|
|
92
100
|
searchType: dataset,
|
|
93
101
|
rootVariable: 'root',
|
|
94
102
|
itemSelector: selectByClass(dataset),
|
|
95
|
-
readers: new SparqlConstructReader({ query: '…' }),
|
|
96
103
|
},
|
|
97
104
|
{
|
|
98
105
|
searchType: organization,
|
|
99
106
|
rootVariable: 'root',
|
|
100
107
|
itemSelector: selectByClass(organization),
|
|
101
|
-
readers: new SparqlConstructReader({ query: '…' }),
|
|
102
108
|
},
|
|
103
109
|
],
|
|
104
110
|
}),
|
|
@@ -180,3 +186,37 @@ across a dataset or across datasets, so the same pipeline indexes a 15-quad
|
|
|
180
186
|
catalog entry and a dataset of millions of objects. The one irreducible atom is a
|
|
181
187
|
single root’s quads: unbounded for a pathological root, and nothing here changes
|
|
182
188
|
that.
|
|
189
|
+
|
|
190
|
+
## Extraction queries and non-deduplicating engines
|
|
191
|
+
|
|
192
|
+
The generated extraction CONSTRUCTs are deliberately shaped to be well-formed for
|
|
193
|
+
**non-deduplicating** SPARQL engines such as
|
|
194
|
+
[QLever](https://github.com/ad-freiburg/qlever), which emit one copy of the
|
|
195
|
+
CONSTRUCT template per solution row rather than a deduplicated set. The generator
|
|
196
|
+
guarantees one output triple per genuine value:
|
|
197
|
+
|
|
198
|
+
- **UNION per field, never conjunction.** Each field is its own `UNION` branch
|
|
199
|
+
(`?root <path> ?value`), so two independent multi-valued fields never form a
|
|
200
|
+
cross-product – the failure mode that inflates a non-deduplicating engine’s
|
|
201
|
+
output by the product of their cardinalities (roughly 18-fold on a measured
|
|
202
|
+
QLever workload).
|
|
203
|
+
- **No projected-away template triple.** Every template triple binds its own
|
|
204
|
+
variables (`?root <alias> ?value`); there is no constant triple (e.g.
|
|
205
|
+
`?root a <Type>`) to be re-emitted once per solution.
|
|
206
|
+
- **Given roots.** Roots arrive as a `VALUES` set, not a pattern that fans out.
|
|
207
|
+
- **Inline references keep the discipline recursively:** a reference type’s
|
|
208
|
+
fields are UNION’d off the referent variable, so even a multi-hop nested
|
|
209
|
+
template never conjoins independent multi-valued fields – only the intermediate
|
|
210
|
+
link triple repeats, and only linearly.
|
|
211
|
+
|
|
212
|
+
Because the queries are duplicate-free by construction, correctness and bounded
|
|
213
|
+
output volume do **not** depend on a client-side **post-processing deduplication
|
|
214
|
+
step** – buffering a whole result to deduplicate it would defeat the
|
|
215
|
+
batch-bounded streaming memory model above. The only residual duplication is
|
|
216
|
+
linear and comes from duplicate _input_ – a non-`DISTINCT` selector feeding a
|
|
217
|
+
multi-typed root more than once – which the streaming per-quad subject index
|
|
218
|
+
(`buildSubjectIndex`) collapses as a cheap backstop, not as a compensator for
|
|
219
|
+
query-shape inflation. Validated on a real QLever index over the full source
|
|
220
|
+
dump: raw CONSTRUCT output equalled the distinct set for a type with distinct
|
|
221
|
+
roots, and the projected documents were byte-identical to those from a
|
|
222
|
+
deduplicating engine.
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
import { type QueryConstruct } from '@traqula/rules-sparql-1-1';
|
|
2
|
+
import type { SearchSchema, SearchType } from '@lde/search';
|
|
3
|
+
/** Options for {@link extractionQuery}. */
|
|
4
|
+
export interface ExtractionOptions {
|
|
5
|
+
/**
|
|
6
|
+
* The variable the roots bind to. Left free in the generated query for the
|
|
7
|
+
* pipeline’s VALUES injection (`injectValues`), so it must match the stage’s
|
|
8
|
+
* `rootVariable` – the variable the item selector projects. Defaults to
|
|
9
|
+
* `root`, matching `selectByClass`. Must not be `dataset` (the SPARQL reader
|
|
10
|
+
* substitutes `?dataset` with the dataset IRI).
|
|
11
|
+
* @default 'root'
|
|
12
|
+
*/
|
|
13
|
+
readonly subjectVariable?: string;
|
|
14
|
+
}
|
|
15
|
+
/**
|
|
16
|
+
* Generate a type’s **Extraction** CONSTRUCT from its {@link SearchType}
|
|
17
|
+
* declaration: the query whose output the projection frames and reads. Pure –
|
|
18
|
+
* `SearchType → QueryConstruct` – with no engine or deployment knowledge.
|
|
19
|
+
*
|
|
20
|
+
* - **template**: one `?root <`{@link irAlias IR Alias}`> ?value` triple per
|
|
21
|
+
* path-bearing field – IRIs only, a single subject. A property path cannot be
|
|
22
|
+
* a CONSTRUCT verb, so the flattened value must be minted onto its subject
|
|
23
|
+
* under the field’s alias; the projection reads it back under the same alias.
|
|
24
|
+
* - **WHERE**: one UNION branch per field (`?root <path> ?value`), the field’s
|
|
25
|
+
* `path` embedded as a literal SPARQL property path (sequence / alternative /
|
|
26
|
+
* inverse). UNION per branch, never a conjunction – conjoining every path in
|
|
27
|
+
* one BGP cross-multiplies the multi-valued ones (~4× inflation).
|
|
28
|
+
* - **roots**: the subject variable is left free, to be bound by the pipeline’s
|
|
29
|
+
* `injectValues` (`VALUES ?root { … }`). Root *selection* is a separate,
|
|
30
|
+
* deployment concern (`selectByClass` / the stage’s `itemSelector`); this
|
|
31
|
+
* generator emits IR Aliases, the selector queries the source class.
|
|
32
|
+
* - **inline references** ({@link isInlineReference}): one CONSTRUCT with a
|
|
33
|
+
* nested template (`{ ?root <…/ref> ?r . ?r <…/field> ?v }`), recursing into
|
|
34
|
+
* the reference type to the schema’s declared depth. The referent-binding hop
|
|
35
|
+
* uses the source `path`; the emitted triples use the minted aliases.
|
|
36
|
+
*
|
|
37
|
+
* Wire the result into a `SparqlConstructReader` (see `searchStages`), which
|
|
38
|
+
* runs it per batch with the roots injected as VALUES.
|
|
39
|
+
*/
|
|
40
|
+
export declare function extractionQuery(searchType: SearchType, schema: SearchSchema, options?: ExtractionOptions): QueryConstruct;
|
|
41
|
+
/** {@link extractionQuery}, serialised to a SPARQL string a reader can run. */
|
|
42
|
+
export declare function extractionQueryString(searchType: SearchType, schema: SearchSchema, options?: ExtractionOptions): string;
|
|
43
|
+
//# sourceMappingURL=extraction.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"extraction.d.ts","sourceRoot":"","sources":["../src/extraction.ts"],"names":[],"mappings":"AAAA,OAAO,EAKL,KAAK,cAAc,EAIpB,MAAM,2BAA2B,CAAC;AAQnC,OAAO,KAAK,EAAE,YAAY,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAM5D,2CAA2C;AAC3C,MAAM,WAAW,iBAAiB;IAChC;;;;;;;OAOG;IACH,QAAQ,CAAC,eAAe,CAAC,EAAE,MAAM,CAAC;CACnC;AAED;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AACH,wBAAgB,eAAe,CAC7B,UAAU,EAAE,UAAU,EACtB,MAAM,EAAE,YAAY,EACpB,OAAO,GAAE,iBAAsB,GAC9B,cAAc,CAqBhB;AAED,+EAA+E;AAC/E,wBAAgB,qBAAqB,CACnC,UAAU,EAAE,UAAU,EACtB,MAAM,EAAE,YAAY,EACpB,OAAO,CAAC,EAAE,iBAAiB,GAC1B,MAAM,CAER"}
|
|
@@ -0,0 +1,108 @@
|
|
|
1
|
+
import { AstFactory, } from '@traqula/rules-sparql-1-1';
|
|
2
|
+
import { Parser } from '@traqula/parser-sparql-1-1';
|
|
3
|
+
import { Generator } from '@traqula/generator-sparql-1-1';
|
|
4
|
+
import { irAlias, isInlineReference, referenceTypeNamed, } from '@lde/search/adapter';
|
|
5
|
+
const factory = new AstFactory();
|
|
6
|
+
const parser = new Parser();
|
|
7
|
+
const generator = new Generator();
|
|
8
|
+
/**
|
|
9
|
+
* Generate a type’s **Extraction** CONSTRUCT from its {@link SearchType}
|
|
10
|
+
* declaration: the query whose output the projection frames and reads. Pure –
|
|
11
|
+
* `SearchType → QueryConstruct` – with no engine or deployment knowledge.
|
|
12
|
+
*
|
|
13
|
+
* - **template**: one `?root <`{@link irAlias IR Alias}`> ?value` triple per
|
|
14
|
+
* path-bearing field – IRIs only, a single subject. A property path cannot be
|
|
15
|
+
* a CONSTRUCT verb, so the flattened value must be minted onto its subject
|
|
16
|
+
* under the field’s alias; the projection reads it back under the same alias.
|
|
17
|
+
* - **WHERE**: one UNION branch per field (`?root <path> ?value`), the field’s
|
|
18
|
+
* `path` embedded as a literal SPARQL property path (sequence / alternative /
|
|
19
|
+
* inverse). UNION per branch, never a conjunction – conjoining every path in
|
|
20
|
+
* one BGP cross-multiplies the multi-valued ones (~4× inflation).
|
|
21
|
+
* - **roots**: the subject variable is left free, to be bound by the pipeline’s
|
|
22
|
+
* `injectValues` (`VALUES ?root { … }`). Root *selection* is a separate,
|
|
23
|
+
* deployment concern (`selectByClass` / the stage’s `itemSelector`); this
|
|
24
|
+
* generator emits IR Aliases, the selector queries the source class.
|
|
25
|
+
* - **inline references** ({@link isInlineReference}): one CONSTRUCT with a
|
|
26
|
+
* nested template (`{ ?root <…/ref> ?r . ?r <…/field> ?v }`), recursing into
|
|
27
|
+
* the reference type to the schema’s declared depth. The referent-binding hop
|
|
28
|
+
* uses the source `path`; the emitted triples use the minted aliases.
|
|
29
|
+
*
|
|
30
|
+
* Wire the result into a `SparqlConstructReader` (see `searchStages`), which
|
|
31
|
+
* runs it per batch with the roots injected as VALUES.
|
|
32
|
+
*/
|
|
33
|
+
export function extractionQuery(searchType, schema, options = {}) {
|
|
34
|
+
const subjectVariable = options.subjectVariable ?? 'root';
|
|
35
|
+
const subject = factory.termVariable(subjectVariable, factory.gen());
|
|
36
|
+
const built = buildFor(searchType, subject, schema, { next: 0 });
|
|
37
|
+
if (built.branches.length === 0) {
|
|
38
|
+
throw new Error(`Cannot generate an extraction CONSTRUCT for “${searchType.name}”: it declares no path-bearing field, so there is nothing to extract.`);
|
|
39
|
+
}
|
|
40
|
+
const where = factory.patternGroup([factory.patternUnion(built.branches, factory.gen())], factory.gen());
|
|
41
|
+
return factory.queryConstruct(factory.gen(), [], factory.patternBgp(built.template, factory.gen()), where, {}, factory.datasetClauses([], factory.gen()));
|
|
42
|
+
}
|
|
43
|
+
/** {@link extractionQuery}, serialised to a SPARQL string a reader can run. */
|
|
44
|
+
export function extractionQueryString(searchType, schema, options) {
|
|
45
|
+
return generator.generate(extractionQuery(searchType, schema, options));
|
|
46
|
+
}
|
|
47
|
+
/**
|
|
48
|
+
* Walk a type’s path-bearing fields off a subject variable, collecting the
|
|
49
|
+
* template triples (minted aliases) and one WHERE branch per field (source
|
|
50
|
+
* paths). Recurses into an inline reference’s type off a fresh referent
|
|
51
|
+
* variable, so the nested template keeps the `subject → referent → value` link
|
|
52
|
+
* in one CONSTRUCT. The recursion terminates because `searchSchema` rejects
|
|
53
|
+
* inline reference cycles.
|
|
54
|
+
*/
|
|
55
|
+
function buildFor(searchType, subject, schema, counter) {
|
|
56
|
+
const template = [];
|
|
57
|
+
const branches = [];
|
|
58
|
+
for (const field of searchType.fields) {
|
|
59
|
+
if (field.path === undefined) {
|
|
60
|
+
continue;
|
|
61
|
+
}
|
|
62
|
+
const alias = factory.termNamed(factory.gen(), irAlias(searchType, field));
|
|
63
|
+
const sourcePath = liftPath(field.path);
|
|
64
|
+
if (isInlineReference(field)) {
|
|
65
|
+
const referenceType = referenceTypeNamed(schema, field.ref.typeName);
|
|
66
|
+
if (referenceType === undefined) {
|
|
67
|
+
continue;
|
|
68
|
+
}
|
|
69
|
+
const referent = factory.termVariable(`r${counter.next++}`, factory.gen());
|
|
70
|
+
template.push(factory.triple(subject, alias, referent));
|
|
71
|
+
const nested = buildFor(referenceType, referent, schema, counter);
|
|
72
|
+
template.push(...nested.template);
|
|
73
|
+
const patterns = [
|
|
74
|
+
factory.patternBgp([factory.triple(subject, sourcePath, referent)], factory.gen()),
|
|
75
|
+
];
|
|
76
|
+
// A single-branch union serialises to just its group, so only add the
|
|
77
|
+
// nested union when the reference type has fields of its own.
|
|
78
|
+
if (nested.branches.length > 0) {
|
|
79
|
+
patterns.push(factory.patternUnion(nested.branches, factory.gen()));
|
|
80
|
+
}
|
|
81
|
+
branches.push(factory.patternGroup(patterns, factory.gen()));
|
|
82
|
+
}
|
|
83
|
+
else {
|
|
84
|
+
const value = factory.termVariable(`v${counter.next++}`, factory.gen());
|
|
85
|
+
template.push(factory.triple(subject, alias, value));
|
|
86
|
+
branches.push(factory.patternGroup([
|
|
87
|
+
factory.patternBgp([factory.triple(subject, sourcePath, value)], factory.gen()),
|
|
88
|
+
], factory.gen()));
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
return { template, branches };
|
|
92
|
+
}
|
|
93
|
+
/**
|
|
94
|
+
* Lift a field’s `path` – written in the SPARQL reader adapter’s grammar (a
|
|
95
|
+
* property path) – into a predicate AST node, by parsing it inside a throwaway
|
|
96
|
+
* query and taking the verb. A single IRI yields a plain named node; a
|
|
97
|
+
* sequence / alternative / inverse yields a path node. The WHERE consumes it;
|
|
98
|
+
* the CONSTRUCT template never does (a path cannot be a template verb).
|
|
99
|
+
*/
|
|
100
|
+
function liftPath(path) {
|
|
101
|
+
// Always a single-triple SELECT by construction, so the verb is at a fixed
|
|
102
|
+
// spot; a malformed `path` makes the parser itself throw, which is the right
|
|
103
|
+
// failure for invalid reader-adapter grammar.
|
|
104
|
+
const ast = parser.parse(`SELECT * WHERE { ?s ${path} ?o }`);
|
|
105
|
+
const bgp = ast.where.patterns[0];
|
|
106
|
+
const triple = bgp.triples[0];
|
|
107
|
+
return triple.predicate;
|
|
108
|
+
}
|
package/dist/index.d.ts
CHANGED
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
export { searchStages, selectByClass } from './search-stages.js';
|
|
2
2
|
export type { SearchStagesOptions, SearchStageType } from './search-stages.js';
|
|
3
|
+
export { extractionQuery, extractionQueryString } from './extraction.js';
|
|
4
|
+
export type { ExtractionOptions } from './extraction.js';
|
|
3
5
|
export { searchIndexWriter } from './search-index-writer.js';
|
|
4
6
|
export type { SearchIndexWriterOptions } from './search-index-writer.js';
|
|
5
7
|
export type { TypedSearchDocument } from './typed-search-document.js';
|
package/dist/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AACjE,YAAY,EAAE,mBAAmB,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAC;AAC/E,OAAO,EAAE,iBAAiB,EAAE,MAAM,0BAA0B,CAAC;AAC7D,YAAY,EAAE,wBAAwB,EAAE,MAAM,0BAA0B,CAAC;AACzE,YAAY,EAAE,mBAAmB,EAAE,MAAM,4BAA4B,CAAC"}
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AACjE,YAAY,EAAE,mBAAmB,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAC;AAC/E,OAAO,EAAE,eAAe,EAAE,qBAAqB,EAAE,MAAM,iBAAiB,CAAC;AACzE,YAAY,EAAE,iBAAiB,EAAE,MAAM,iBAAiB,CAAC;AACzD,OAAO,EAAE,iBAAiB,EAAE,MAAM,0BAA0B,CAAC;AAC7D,YAAY,EAAE,wBAAwB,EAAE,MAAM,0BAA0B,CAAC;AACzE,YAAY,EAAE,mBAAmB,EAAE,MAAM,4BAA4B,CAAC"}
|
package/dist/index.js
CHANGED
package/dist/search-stages.d.ts
CHANGED
|
@@ -24,8 +24,16 @@ export interface SearchStageType {
|
|
|
24
24
|
* convenience for the object grain, not a default.
|
|
25
25
|
*/
|
|
26
26
|
itemSelector: ItemSelector;
|
|
27
|
-
/**
|
|
28
|
-
|
|
27
|
+
/**
|
|
28
|
+
* Reader(s) that extract each selected root’s quads. Defaults to a single
|
|
29
|
+
* {@link SparqlConstructReader} running the **Extraction** CONSTRUCT generated
|
|
30
|
+
* from `searchType` ({@link extractionQuery}), with `rootVariable` as the free
|
|
31
|
+
* subject the batch’s roots bind to – the schema-derived reader the projection
|
|
32
|
+
* is guaranteed to agree with (they share the {@link irAlias} convention).
|
|
33
|
+
* Supply your own only to read from a non-SPARQL source, merge several
|
|
34
|
+
* readers, or attach transforms.
|
|
35
|
+
*/
|
|
36
|
+
readers?: StageReaders;
|
|
29
37
|
/**
|
|
30
38
|
* Roots (and so documents) per batch – the memory bound. Under a root-bound
|
|
31
39
|
* selector it moves memory and request count, never output.
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"search-stages.d.ts","sourceRoot":"","sources":["../src/search-stages.ts"],"names":[],"mappings":"AAAA,OAAO,
|
|
1
|
+
{"version":3,"file":"search-stages.d.ts","sourceRoot":"","sources":["../src/search-stages.ts"],"names":[],"mappings":"AAAA,OAAO,EAGL,KAAK,EACL,KAAK,YAAY,EACjB,KAAK,YAAY,EAClB,MAAM,eAAe,CAAC;AACvB,OAAO,EAAgB,KAAK,QAAQ,EAAE,KAAK,YAAY,EAAE,MAAM,aAAa,CAAC;AAE7E,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,4BAA4B,CAAC;AAEtE,kDAAkD;AAClD,MAAM,WAAW,eAAe;IAC9B;;;;;;OAMG;IACH,UAAU,EAAE,QAAQ,CAAC;IACrB;;;;;OAKG;IACH,YAAY,EAAE,MAAM,CAAC;IACrB;;;;OAIG;IACH,YAAY,EAAE,YAAY,CAAC;IAC3B;;;;;;;;OAQG;IACH,OAAO,CAAC,EAAE,YAAY,CAAC;IACvB;;;;OAIG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,8EAA8E;IAC9E,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB;;;;OAIG;IACH,aAAa,CAAC,EAAE,MAAM,CAAC;CACxB;AAED,wCAAwC;AACxC,MAAM,WAAW,mBAAmB;IAClC;;;OAGG;IACH,MAAM,EAAE,YAAY,CAAC;IACrB,4DAA4D;IAC5D,KAAK,EAAE,SAAS,eAAe,EAAE,CAAC;CACnC;AAED;;;;;;;;;;;;;GAaG;AACH,wBAAgB,YAAY,CAC1B,OAAO,EAAE,mBAAmB,GAC3B,KAAK,CAAC,mBAAmB,CAAC,EAAE,CAwD9B;AAED;;;;;;;;;;;;GAYG;AACH,wBAAgB,aAAa,CAC3B,UAAU,EAAE,QAAQ,EACpB,YAAY,SAAS,GACpB,YAAY,CAId"}
|
package/dist/search-stages.js
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
|
-
import { SparqlItemSelector, Stage, } from '@lde/pipeline';
|
|
1
|
+
import { SparqlConstructReader, SparqlItemSelector, Stage, } from '@lde/pipeline';
|
|
2
2
|
import { projectRoots } from '@lde/search';
|
|
3
|
+
import { extractionQueryString } from './extraction.js';
|
|
3
4
|
/**
|
|
4
5
|
* Compose one projecting {@link Stage} per root type – the source side of a
|
|
5
6
|
* search pipeline. Each stage selects its own roots, extracts each root’s quads,
|
|
@@ -25,9 +26,18 @@ export function searchStages(options) {
|
|
|
25
26
|
throw new Error(`Search type “${type.searchType.name}” (class ${type.searchType.class}) is not in the schema; searchStages projects only types the schema declares.`);
|
|
26
27
|
}
|
|
27
28
|
const { rootVariable } = type;
|
|
29
|
+
// Default to the Extraction CONSTRUCT generated from the schema, its subject
|
|
30
|
+
// left free for the batch’s VALUES injection. The reader and the projection
|
|
31
|
+
// then agree by construction: both key off the same IR Aliases.
|
|
32
|
+
const readers = type.readers ??
|
|
33
|
+
new SparqlConstructReader({
|
|
34
|
+
query: extractionQueryString(searchType, schema, {
|
|
35
|
+
subjectVariable: rootVariable,
|
|
36
|
+
}),
|
|
37
|
+
});
|
|
28
38
|
return new Stage({
|
|
29
39
|
name: searchType.name,
|
|
30
|
-
readers
|
|
40
|
+
readers,
|
|
31
41
|
itemSelector: type.itemSelector,
|
|
32
42
|
batchSize: type.batchSize,
|
|
33
43
|
maxConcurrency: type.maxConcurrency,
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@lde/search-pipeline",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.11.0",
|
|
4
4
|
"description": "Search indexing as an @lde/pipeline instance: a Writer<Quad> that applies the @lde/search projection to extracted RDF and fans each document out to the transactional engine writer for its type's collection. Glue between the pipeline quad world and the search-engine document world; owns no projection or engine logic itself.",
|
|
5
5
|
"repository": {
|
|
6
6
|
"url": "git+https://github.com/ldelements/lde.git",
|
|
@@ -25,11 +25,14 @@
|
|
|
25
25
|
"!**/*.tsbuildinfo"
|
|
26
26
|
],
|
|
27
27
|
"dependencies": {
|
|
28
|
-
"@lde/search": "^0.
|
|
28
|
+
"@lde/search": "^0.12.0",
|
|
29
|
+
"@traqula/generator-sparql-1-1": "^1.1.8",
|
|
30
|
+
"@traqula/parser-sparql-1-1": "^1.1.5",
|
|
31
|
+
"@traqula/rules-sparql-1-1": "^1.1.0",
|
|
29
32
|
"tslib": "^2.3.0"
|
|
30
33
|
},
|
|
31
34
|
"devDependencies": {
|
|
32
|
-
"@lde/search-typesense": "^0.
|
|
35
|
+
"@lde/search-typesense": "^0.14.0",
|
|
33
36
|
"@rdfjs/types": "^2.0.1",
|
|
34
37
|
"n3": "^2.1.1",
|
|
35
38
|
"testcontainers": "^12.0.3",
|