@lde/search-typesense 0.7.1 → 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +18 -15
- package/dist/blue-green-rebuild.d.ts +1 -1
- package/dist/blue-green-rebuild.d.ts.map +1 -1
- package/dist/blue-green-rebuild.js +5 -5
- package/dist/{collection-schema.d.ts → collection-definition.d.ts} +11 -10
- package/dist/collection-definition.d.ts.map +1 -0
- package/dist/{collection-schema.js → collection-definition.js} +27 -19
- package/dist/in-place-rebuild.d.ts.map +1 -1
- package/dist/in-place-rebuild.js +5 -5
- package/dist/index.d.ts +2 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -1
- package/dist/rebuild-support.d.ts +5 -5
- package/dist/rebuild-support.d.ts.map +1 -1
- package/dist/rebuild-support.js +7 -2
- package/dist/search.d.ts +13 -12
- package/dist/search.d.ts.map +1 -1
- package/dist/search.js +41 -47
- package/package.json +2 -2
- package/dist/collection-schema.d.ts.map +0 -1
package/README.md
CHANGED
|
@@ -4,32 +4,35 @@
|
|
|
4
4
|
domain-agnostic [`@lde/search`](../search) core. **Engine-specific (Typesense) but
|
|
5
5
|
domain-agnostic** – you supply a `SearchType`; this package never names your
|
|
6
6
|
domain. It is the Typesense implementation of the `SearchEngine` port: it derives
|
|
7
|
-
a collection
|
|
7
|
+
a collection definition from the field model, compiles the neutral `SearchQuery` into
|
|
8
8
|
Typesense search params, runs it, reconstructs the engine-neutral `SearchResult`,
|
|
9
9
|
and manages the search index lifecycle as transactional `@lde/pipeline`
|
|
10
10
|
writers (Blue/green Rebuild and In-place Rebuild).
|
|
11
11
|
|
|
12
12
|
## Collection schema and engine
|
|
13
13
|
|
|
14
|
-
`
|
|
15
|
-
Typesense collection from the unified `SearchField` model
|
|
14
|
+
`buildCollectionDefinition(searchType, { name, defaultSortingField, … })` derives a
|
|
15
|
+
Typesense collection from the unified `SearchField` model – the Typesense field
|
|
16
16
|
type comes from each field’s `kind`, and the physical fanout (per-locale
|
|
17
|
-
search/sort keys
|
|
18
|
-
`@lde/search`’s `physicalFields
|
|
17
|
+
search/sort keys, plus one regex display field per output text field) matches
|
|
18
|
+
what the projection writes, via `@lde/search`’s `physicalFields` and its display
|
|
19
|
+
helpers, so the index and the documents cannot drift.
|
|
19
20
|
|
|
20
21
|
**Memory lever.** Typesense keeps the index in RAM (with a raw copy of each
|
|
21
22
|
document on disk), so RAM tracks the _indexed_ surface – roughly 2–3× the size
|
|
22
23
|
of the fields you search, facet or sort on – not the full document.
|
|
23
|
-
`
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
24
|
+
`buildCollectionDefinition` keeps that surface minimal: the `output` display
|
|
25
|
+
labels land in one `index: false` regex field (`${name}_<lang>`, one value per
|
|
26
|
+
present language), stored on disk and fetched only for a hit, so they cost no RAM
|
|
27
|
+
and preserve every language; only the folded `*_search_${locale}`,
|
|
28
|
+
facet/reference and `*_sort_${locale}` companions are indexed. Keeping
|
|
29
|
+
retrieval-only fields un-indexed is the lever for holding a large index’s RAM
|
|
30
|
+
down.
|
|
28
31
|
|
|
29
32
|
`createTypesenseSearchEngine(client, schema, { collections })` is the
|
|
30
33
|
`SearchEngine` implementation. Each search:
|
|
31
34
|
|
|
32
|
-
- validates the query against the search type (the port contract
|
|
35
|
+
- validates the query against the search type (the port contract – a
|
|
33
36
|
structurally invalid query is rejected, never sent);
|
|
34
37
|
- compiles it into Typesense search params (`buildSearchParams`);
|
|
35
38
|
- runs the search;
|
|
@@ -38,7 +41,7 @@ un-indexed is the lever for holding a large index’s RAM down.
|
|
|
38
41
|
sources bundled into a single lookup. A reference without a `labelSource`
|
|
39
42
|
stays id-only. With `labelCacheTtlMs` set, each label-source collection is
|
|
40
43
|
instead loaded once into an in-memory cache;
|
|
41
|
-
- reconstructs the logical `SearchResult` (`parseSearchResponse`)
|
|
44
|
+
- reconstructs the logical `SearchResult` (`parseSearchResponse`) – language
|
|
42
45
|
maps, labelled references, labelled facet buckets.
|
|
43
46
|
|
|
44
47
|
A label source is just another `SearchType` in the schema (with an `output`,
|
|
@@ -67,9 +70,9 @@ Indexing runs through two transactional writers, one per update mode – the
|
|
|
67
70
|
Both implement `@lde/pipeline`’s `Writer` – each run is
|
|
68
71
|
`openRun(context)` → `write` per dataset → `commit()` or `abort(error)` – so
|
|
69
72
|
an `@lde/pipeline` `Pipeline` drives them without branching on the mode. Both
|
|
70
|
-
derive the collection
|
|
71
|
-
`
|
|
72
|
-
`
|
|
73
|
+
derive the collection definition from your `SearchType` (via
|
|
74
|
+
`buildCollectionDefinition`), and their options accept everything
|
|
75
|
+
`buildCollectionDefinition` does (`defaultLocale`, `defaultSortingField`,
|
|
73
76
|
`synonymSets`) plus the tuning knobs (`batchSize`, `lockTtlMs`).
|
|
74
77
|
|
|
75
78
|
### Blue/green Rebuild
|
|
@@ -2,7 +2,7 @@ import type { Client } from 'typesense';
|
|
|
2
2
|
import type { SearchType } from '@lde/search';
|
|
3
3
|
import type { RunContext, RunWriter, Writer } from '@lde/pipeline';
|
|
4
4
|
import { type RebuildOptions } from './rebuild-support.js';
|
|
5
|
-
/** {@link BlueGreenRebuild} options: the collection-
|
|
5
|
+
/** {@link BlueGreenRebuild} options: the collection-definition options (`name` is
|
|
6
6
|
* the logical index name the alias is kept on) plus the rebuild tuning knobs. */
|
|
7
7
|
export type BlueGreenRebuildOptions = RebuildOptions;
|
|
8
8
|
/**
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"blue-green-rebuild.d.ts","sourceRoot":"","sources":["../src/blue-green-rebuild.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,WAAW,CAAC;AACxC,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAC9C,OAAO,KAAK,EAEV,UAAU,EACV,SAAS,EACT,MAAM,EACP,MAAM,eAAe,CAAC;AAMvB,OAAO,EAKL,KAAK,cAAc,EACpB,MAAM,sBAAsB,CAAC;AAE9B;kFACkF;AAClF,MAAM,MAAM,uBAAuB,GAAG,cAAc,CAAC;AAErD;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AACH,qBAAa,gBAAgB,CAC3B,SAAS,SAAS;IAAE,EAAE,EAAE,MAAM,CAAA;CAAE,CAChC,YAAW,MAAM,CAAC,SAAS,CAAC;IAE1B,OAAO,CAAC,QAAQ,CAAC,MAAM;IACvB,OAAO,CAAC,QAAQ,CAAC,UAAU;IAC3B,OAAO,CAAC,QAAQ,CAAC,OAAO;gBAFP,MAAM,EAAE,MAAM,EACd,UAAU,EAAE,UAAU,EACtB,OAAO,EAAE,uBAAuB;IAM7C,OAAO,CAAC,OAAO,EAAE,UAAU,GAAG,OAAO,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC;
|
|
1
|
+
{"version":3,"file":"blue-green-rebuild.d.ts","sourceRoot":"","sources":["../src/blue-green-rebuild.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,WAAW,CAAC;AACxC,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAC9C,OAAO,KAAK,EAEV,UAAU,EACV,SAAS,EACT,MAAM,EACP,MAAM,eAAe,CAAC;AAMvB,OAAO,EAKL,KAAK,cAAc,EACpB,MAAM,sBAAsB,CAAC;AAE9B;kFACkF;AAClF,MAAM,MAAM,uBAAuB,GAAG,cAAc,CAAC;AAErD;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AACH,qBAAa,gBAAgB,CAC3B,SAAS,SAAS;IAAE,EAAE,EAAE,MAAM,CAAA;CAAE,CAChC,YAAW,MAAM,CAAC,SAAS,CAAC;IAE1B,OAAO,CAAC,QAAQ,CAAC,MAAM;IACvB,OAAO,CAAC,QAAQ,CAAC,UAAU;IAC3B,OAAO,CAAC,QAAQ,CAAC,OAAO;gBAFP,MAAM,EAAE,MAAM,EACd,UAAU,EAAE,UAAU,EACtB,OAAO,EAAE,uBAAuB;IAM7C,OAAO,CAAC,OAAO,EAAE,UAAU,GAAG,OAAO,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC;IA6FjE,4EAA4E;YAC9D,WAAW;CAW1B"}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { buildCollectionDefinition } from './collection-definition.js';
|
|
2
2
|
import { BatchImporter } from './import.js';
|
|
3
3
|
import { httpStatus, openLockedRun, releaseLock } from './lock.js';
|
|
4
4
|
import { SOURCE_FIELD, sourceDocumentsFilter } from './sweep.js';
|
|
@@ -42,19 +42,19 @@ export class BlueGreenRebuild {
|
|
|
42
42
|
assertNoReservedFields(searchType, [SOURCE_FIELD]);
|
|
43
43
|
}
|
|
44
44
|
async openRun(context) {
|
|
45
|
-
const { name, batchSize, lockTtlMs,
|
|
45
|
+
const { name, batchSize, lockTtlMs, definitionOptions } = resolveRebuildOptions(this.options);
|
|
46
46
|
return openLockedRun(this.client, name, lockTtlMs, async () => {
|
|
47
47
|
// Create the fresh (blue) collection up front, so a failure surfaces
|
|
48
48
|
// before any dataset is processed. startedAt orders the versioned names;
|
|
49
49
|
// concurrent same-name runs are excluded by the lock.
|
|
50
50
|
const collection = `${name}_${Date.parse(context.startedAt)}`;
|
|
51
51
|
const previous = await this.aliasTarget(name);
|
|
52
|
-
const
|
|
52
|
+
const definition = buildCollectionDefinition(this.searchType, definitionOptions);
|
|
53
53
|
await this.client.collections().create({
|
|
54
|
-
...
|
|
54
|
+
...definition,
|
|
55
55
|
name: collection,
|
|
56
56
|
fields: [
|
|
57
|
-
...(
|
|
57
|
+
...(definition.fields ?? []),
|
|
58
58
|
{ name: SOURCE_FIELD, type: 'string' },
|
|
59
59
|
],
|
|
60
60
|
});
|
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
import type { CollectionCreateSchema } from 'typesense';
|
|
2
2
|
import { type SearchType } from '@lde/search';
|
|
3
3
|
/** Deployment-specific options the generic field model does not carry. */
|
|
4
|
-
export interface
|
|
4
|
+
export interface CollectionDefinitionOptions {
|
|
5
5
|
/** The Typesense collection (or alias) name. */
|
|
6
6
|
readonly name: string;
|
|
7
7
|
/** Snowball stemming locale for non-localized searchable fields (e.g. `en`).
|
|
8
|
-
* Unset, those fields are not stemmed
|
|
8
|
+
* Unset, those fields are not stemmed – folding still applies – so no
|
|
9
9
|
* language is ever assumed. Localized text search fields always stem in
|
|
10
10
|
* their own locale. */
|
|
11
11
|
readonly defaultLocale?: string;
|
|
@@ -15,7 +15,7 @@ export interface CollectionSchemaOptions {
|
|
|
15
15
|
readonly synonymSets?: readonly string[];
|
|
16
16
|
}
|
|
17
17
|
/**
|
|
18
|
-
* Build a Typesense collection
|
|
18
|
+
* Build a Typesense collection definition from the unified {@link SearchType}, so
|
|
19
19
|
* the index and the projection are driven by one declarative source and cannot
|
|
20
20
|
* drift. Each field fans out into the same physical fields the projection writes
|
|
21
21
|
* ({@link physicalFields}); the Typesense field type is derived from the field
|
|
@@ -29,11 +29,12 @@ export interface CollectionSchemaOptions {
|
|
|
29
29
|
* Memory lever: Typesense holds the index in RAM (with a raw copy of each
|
|
30
30
|
* document on disk), so RAM tracks the *indexed* surface – roughly 2–3× the
|
|
31
31
|
* size of the fields you search, facet or sort on – not the whole document.
|
|
32
|
-
* This builder keeps that surface minimal: the `output` display labels
|
|
33
|
-
*
|
|
34
|
-
*
|
|
35
|
-
* `*
|
|
36
|
-
* un-indexed is the lever
|
|
32
|
+
* This builder keeps that surface minimal: the `output` display labels land in
|
|
33
|
+
* a single `index: false` regex field (`${name}_<lang>`, one value per present
|
|
34
|
+
* language), kept on disk and read back only for a hit, so they cost no RAM;
|
|
35
|
+
* only the folded `*_search_${locale}`, facet/reference and `*_sort_${locale}`
|
|
36
|
+
* companions are indexed. Keeping retrieval-only fields un-indexed is the lever
|
|
37
|
+
* for holding a large index’s RAM down.
|
|
37
38
|
*/
|
|
38
|
-
export declare function
|
|
39
|
-
//# sourceMappingURL=collection-
|
|
39
|
+
export declare function buildCollectionDefinition(searchType: SearchType, options: CollectionDefinitionOptions): CollectionCreateSchema;
|
|
40
|
+
//# sourceMappingURL=collection-definition.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"collection-definition.d.ts","sourceRoot":"","sources":["../src/collection-definition.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,WAAW,CAAC;AAExD,OAAO,EAAoB,KAAK,UAAU,EAAE,MAAM,aAAa,CAAC;AAGhE,0EAA0E;AAC1E,MAAM,WAAW,2BAA2B;IAC1C,gDAAgD;IAChD,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB;;;4BAGwB;IACxB,QAAQ,CAAC,aAAa,CAAC,EAAE,MAAM,CAAC;IAChC,kEAAkE;IAClE,QAAQ,CAAC,mBAAmB,CAAC,EAAE,MAAM,CAAC;IACtC,kEAAkE;IAClE,QAAQ,CAAC,WAAW,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;CAC1C;AAED;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,wBAAgB,yBAAyB,CACvC,UAAU,EAAE,UAAU,EACtB,OAAO,EAAE,2BAA2B,GACnC,sBAAsB,CAexB"}
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import { physicalFields } from '@lde/search/adapter';
|
|
1
|
+
import { displayFieldPattern, physicalFields } from '@lde/search/adapter';
|
|
2
2
|
/**
|
|
3
|
-
* Build a Typesense collection
|
|
3
|
+
* Build a Typesense collection definition from the unified {@link SearchType}, so
|
|
4
4
|
* the index and the projection are driven by one declarative source and cannot
|
|
5
5
|
* drift. Each field fans out into the same physical fields the projection writes
|
|
6
6
|
* ({@link physicalFields}); the Typesense field type is derived from the field
|
|
@@ -14,13 +14,14 @@ import { physicalFields } from '@lde/search/adapter';
|
|
|
14
14
|
* Memory lever: Typesense holds the index in RAM (with a raw copy of each
|
|
15
15
|
* document on disk), so RAM tracks the *indexed* surface – roughly 2–3× the
|
|
16
16
|
* size of the fields you search, facet or sort on – not the whole document.
|
|
17
|
-
* This builder keeps that surface minimal: the `output` display labels
|
|
18
|
-
*
|
|
19
|
-
*
|
|
20
|
-
* `*
|
|
21
|
-
* un-indexed is the lever
|
|
17
|
+
* This builder keeps that surface minimal: the `output` display labels land in
|
|
18
|
+
* a single `index: false` regex field (`${name}_<lang>`, one value per present
|
|
19
|
+
* language), kept on disk and read back only for a hit, so they cost no RAM;
|
|
20
|
+
* only the folded `*_search_${locale}`, facet/reference and `*_sort_${locale}`
|
|
21
|
+
* companions are indexed. Keeping retrieval-only fields un-indexed is the lever
|
|
22
|
+
* for holding a large index’s RAM down.
|
|
22
23
|
*/
|
|
23
|
-
export function
|
|
24
|
+
export function buildCollectionDefinition(searchType, options) {
|
|
24
25
|
const { defaultLocale } = options;
|
|
25
26
|
const collection = {
|
|
26
27
|
name: options.name,
|
|
@@ -39,17 +40,24 @@ function typesenseFields(field, defaultLocale, defaultSortingField) {
|
|
|
39
40
|
const names = physicalFields(field);
|
|
40
41
|
if (field.kind === 'text') {
|
|
41
42
|
const locales = field.locales;
|
|
43
|
+
const displayPattern = displayFieldPattern(field);
|
|
42
44
|
return [
|
|
43
|
-
// Display labels:
|
|
44
|
-
//
|
|
45
|
-
//
|
|
46
|
-
//
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
45
|
+
// Display labels: ONE regex field (`${name}_<lang>`) storing every
|
|
46
|
+
// present language’s value, NOT indexed (`index: false`) – search hits
|
|
47
|
+
// the folded `*_search` companions, so the display copies stay on disk and
|
|
48
|
+
// off RAM (fetched only for a hit), accents preserved, and a language
|
|
49
|
+
// outside `locales` still renders. This is the memory lever: RAM tracks
|
|
50
|
+
// the search surface, not the display text. Absent for a non-output field.
|
|
51
|
+
...(displayPattern !== undefined
|
|
52
|
+
? [
|
|
53
|
+
{
|
|
54
|
+
name: displayPattern,
|
|
55
|
+
type: 'string',
|
|
56
|
+
index: false,
|
|
57
|
+
optional: true,
|
|
58
|
+
},
|
|
59
|
+
]
|
|
60
|
+
: []),
|
|
53
61
|
// One folded search field per locale, each stemmed in its own
|
|
54
62
|
// language; the untagged `und` locale is folded but unstemmed unless
|
|
55
63
|
// the deployment opts in via `defaultLocale`.
|
|
@@ -83,7 +91,7 @@ function typesenseFields(field, defaultLocale, defaultSortingField) {
|
|
|
83
91
|
},
|
|
84
92
|
];
|
|
85
93
|
// `names.search` is non-empty exactly when the field projects a folded
|
|
86
|
-
// search companion
|
|
94
|
+
// search companion – physicalFields owns that rule.
|
|
87
95
|
for (const name of names.search) {
|
|
88
96
|
fields.push({
|
|
89
97
|
name,
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"in-place-rebuild.d.ts","sourceRoot":"","sources":["../src/in-place-rebuild.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,WAAW,CAAC;AAExC,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAE9C,OAAO,KAAK,EAEV,UAAU,EACV,SAAS,EACT,MAAM,EACP,MAAM,eAAe,CAAC;AAYvB,OAAO,EAKL,KAAK,cAAc,EACpB,MAAM,sBAAsB,CAAC;AAa9B;gCACgC;AAChC,MAAM,WAAW,qBAAsB,SAAQ,cAAc;IAC3D;sEACkE;IAClE,QAAQ,CAAC,mBAAmB,CAAC,EAAE,MAAM,CAAC;CACvC;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgCG;AACH,qBAAa,cAAc,CACzB,SAAS,SAAS;IAAE,EAAE,EAAE,MAAM,CAAA;CAAE,CAChC,YAAW,MAAM,CAAC,SAAS,CAAC;IAE1B,OAAO,CAAC,QAAQ,CAAC,MAAM;IACvB,OAAO,CAAC,QAAQ,CAAC,UAAU;IAC3B,OAAO,CAAC,QAAQ,CAAC,OAAO;gBAFP,MAAM,EAAE,MAAM,EACd,UAAU,EAAE,UAAU,EACtB,OAAO,EAAE,qBAAqB;IAK3C,OAAO,CAAC,OAAO,EAAE,UAAU,GAAG,OAAO,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC;
|
|
1
|
+
{"version":3,"file":"in-place-rebuild.d.ts","sourceRoot":"","sources":["../src/in-place-rebuild.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,WAAW,CAAC;AAExC,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAE9C,OAAO,KAAK,EAEV,UAAU,EACV,SAAS,EACT,MAAM,EACP,MAAM,eAAe,CAAC;AAYvB,OAAO,EAKL,KAAK,cAAc,EACpB,MAAM,sBAAsB,CAAC;AAa9B;gCACgC;AAChC,MAAM,WAAW,qBAAsB,SAAQ,cAAc;IAC3D;sEACkE;IAClE,QAAQ,CAAC,mBAAmB,CAAC,EAAE,MAAM,CAAC;CACvC;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgCG;AACH,qBAAa,cAAc,CACzB,SAAS,SAAS;IAAE,EAAE,EAAE,MAAM,CAAA;CAAE,CAChC,YAAW,MAAM,CAAC,SAAS,CAAC;IAE1B,OAAO,CAAC,QAAQ,CAAC,MAAM;IACvB,OAAO,CAAC,QAAQ,CAAC,UAAU;IAC3B,OAAO,CAAC,QAAQ,CAAC,OAAO;gBAFP,MAAM,EAAE,MAAM,EACd,UAAU,EAAE,UAAU,EACtB,OAAO,EAAE,qBAAqB;IAK3C,OAAO,CAAC,OAAO,EAAE,UAAU,GAAG,OAAO,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC;IA0FjE;;;;;;OAMG;YACW,cAAc;CAsB7B"}
|
package/dist/in-place-rebuild.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { buildCollectionDefinition } from './collection-definition.js';
|
|
2
2
|
import { BatchImporter } from './import.js';
|
|
3
3
|
import { ensureCollectionExists, openLockedRun, releaseLock } from './lock.js';
|
|
4
4
|
import { LAST_SEEN_FIELD, SOURCE_FIELD, departedSources, membershipSweepFilters, staleDocumentsFilter, thisRunDocumentsFilter, } from './sweep.js';
|
|
@@ -58,20 +58,20 @@ export class InPlaceRebuild {
|
|
|
58
58
|
}
|
|
59
59
|
async openRun(context) {
|
|
60
60
|
const { maxSweepableSources = DEFAULT_MAX_SWEEPABLE_SOURCES, ...rebuildOptions } = this.options;
|
|
61
|
-
const { name, batchSize, lockTtlMs,
|
|
61
|
+
const { name, batchSize, lockTtlMs, definitionOptions } = resolveRebuildOptions(rebuildOptions);
|
|
62
62
|
return openLockedRun(this.client, name, lockTtlMs, async () => {
|
|
63
63
|
// Create the collection on demand: SearchType schema + the bookkeeping
|
|
64
64
|
// fields, `source` faceted so the membership sweep can enumerate the
|
|
65
65
|
// distinct sources.
|
|
66
66
|
await ensureCollectionExists(this.client, name, () => {
|
|
67
|
-
const
|
|
67
|
+
const definition = buildCollectionDefinition(this.searchType, definitionOptions);
|
|
68
68
|
const bookkeeping = [
|
|
69
69
|
{ name: SOURCE_FIELD, type: 'string', facet: true },
|
|
70
70
|
{ name: LAST_SEEN_FIELD, type: 'string' },
|
|
71
71
|
];
|
|
72
72
|
return {
|
|
73
|
-
...
|
|
74
|
-
fields: [...(
|
|
73
|
+
...definition,
|
|
74
|
+
fields: [...(definition.fields ?? []), ...bookkeeping],
|
|
75
75
|
};
|
|
76
76
|
});
|
|
77
77
|
const importer = new BatchImporter(this.client, name, batchSize);
|
package/dist/index.d.ts
CHANGED
|
@@ -4,8 +4,8 @@ export { InPlaceRebuild } from './in-place-rebuild.js';
|
|
|
4
4
|
export type { InPlaceRebuildOptions } from './in-place-rebuild.js';
|
|
5
5
|
export { RebuildAlreadyRunning } from './lock.js';
|
|
6
6
|
export { departedSources, membershipSweepFilters, sourceDocumentsFilter, staleDocumentsFilter, } from './sweep.js';
|
|
7
|
-
export {
|
|
8
|
-
export type {
|
|
7
|
+
export { buildCollectionDefinition } from './collection-definition.js';
|
|
8
|
+
export type { CollectionDefinitionOptions } from './collection-definition.js';
|
|
9
9
|
export { buildSearchParams } from './query-compiler.js';
|
|
10
10
|
export type { BuildSearchParamsOptions } from './query-compiler.js';
|
|
11
11
|
export { createTypesenseSearchEngine, parseSearchResponse } from './search.js';
|
package/dist/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,gBAAgB,EAAE,MAAM,yBAAyB,CAAC;AAC3D,YAAY,EAAE,uBAAuB,EAAE,MAAM,yBAAyB,CAAC;AACvE,OAAO,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAC;AACvD,YAAY,EAAE,qBAAqB,EAAE,MAAM,uBAAuB,CAAC;AACnE,OAAO,EAAE,qBAAqB,EAAE,MAAM,WAAW,CAAC;AAClD,OAAO,EACL,eAAe,EACf,sBAAsB,EACtB,qBAAqB,EACrB,oBAAoB,GACrB,MAAM,YAAY,CAAC;AACpB,OAAO,EAAE,
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,gBAAgB,EAAE,MAAM,yBAAyB,CAAC;AAC3D,YAAY,EAAE,uBAAuB,EAAE,MAAM,yBAAyB,CAAC;AACvE,OAAO,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAC;AACvD,YAAY,EAAE,qBAAqB,EAAE,MAAM,uBAAuB,CAAC;AACnE,OAAO,EAAE,qBAAqB,EAAE,MAAM,WAAW,CAAC;AAClD,OAAO,EACL,eAAe,EACf,sBAAsB,EACtB,qBAAqB,EACrB,oBAAoB,GACrB,MAAM,YAAY,CAAC;AACpB,OAAO,EAAE,yBAAyB,EAAE,MAAM,4BAA4B,CAAC;AACvE,YAAY,EAAE,2BAA2B,EAAE,MAAM,4BAA4B,CAAC;AAC9E,OAAO,EAAE,iBAAiB,EAAE,MAAM,qBAAqB,CAAC;AACxD,YAAY,EAAE,wBAAwB,EAAE,MAAM,qBAAqB,CAAC;AACpE,OAAO,EAAE,2BAA2B,EAAE,mBAAmB,EAAE,MAAM,aAAa,CAAC;AAC/E,YAAY,EACV,4BAA4B,EAC5B,uBAAuB,GACxB,MAAM,aAAa,CAAC"}
|
package/dist/index.js
CHANGED
|
@@ -2,6 +2,6 @@ export { BlueGreenRebuild } from './blue-green-rebuild.js';
|
|
|
2
2
|
export { InPlaceRebuild } from './in-place-rebuild.js';
|
|
3
3
|
export { RebuildAlreadyRunning } from './lock.js';
|
|
4
4
|
export { departedSources, membershipSweepFilters, sourceDocumentsFilter, staleDocumentsFilter, } from './sweep.js';
|
|
5
|
-
export {
|
|
5
|
+
export { buildCollectionDefinition } from './collection-definition.js';
|
|
6
6
|
export { buildSearchParams } from './query-compiler.js';
|
|
7
7
|
export { createTypesenseSearchEngine, parseSearchResponse } from './search.js';
|
|
@@ -1,23 +1,23 @@
|
|
|
1
1
|
import type { Client } from 'typesense';
|
|
2
2
|
import type { SearchType } from '@lde/search';
|
|
3
|
-
import type {
|
|
3
|
+
import type { CollectionDefinitionOptions } from './collection-definition.js';
|
|
4
4
|
/**
|
|
5
|
-
* The tuning knobs both rebuild writers share, on top of the collection-
|
|
5
|
+
* The tuning knobs both rebuild writers share, on top of the collection-definition
|
|
6
6
|
* options. Each writer may add its own (In-place adds `maxSweepableSources`).
|
|
7
7
|
*/
|
|
8
|
-
export interface RebuildOptions extends
|
|
8
|
+
export interface RebuildOptions extends CollectionDefinitionOptions {
|
|
9
9
|
/** Documents imported per Typesense request (default 1000). */
|
|
10
10
|
readonly batchSize?: number;
|
|
11
11
|
/** A held lock older than this (ms) is reclaimed (default 10 minutes). */
|
|
12
12
|
readonly lockTtlMs?: number;
|
|
13
13
|
}
|
|
14
14
|
/** The shared options resolved to concrete values plus the residual
|
|
15
|
-
* collection-
|
|
15
|
+
* collection-definition options a writer passes to {@link buildCollectionDefinition}. */
|
|
16
16
|
export interface ResolvedRebuildOptions {
|
|
17
17
|
readonly name: string;
|
|
18
18
|
readonly batchSize: number;
|
|
19
19
|
readonly lockTtlMs: number;
|
|
20
|
-
readonly
|
|
20
|
+
readonly definitionOptions: CollectionDefinitionOptions;
|
|
21
21
|
}
|
|
22
22
|
/** Apply the shared defaults, once, so neither writer restates them. */
|
|
23
23
|
export declare function resolveRebuildOptions(options: RebuildOptions): ResolvedRebuildOptions;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"rebuild-support.d.ts","sourceRoot":"","sources":["../src/rebuild-support.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,WAAW,CAAC;AACxC,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAC9C,OAAO,KAAK,EAAE,
|
|
1
|
+
{"version":3,"file":"rebuild-support.d.ts","sourceRoot":"","sources":["../src/rebuild-support.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,WAAW,CAAC;AACxC,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAC9C,OAAO,KAAK,EAAE,2BAA2B,EAAE,MAAM,4BAA4B,CAAC;AAI9E;;;GAGG;AACH,MAAM,WAAW,cAAe,SAAQ,2BAA2B;IACjE,+DAA+D;IAC/D,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC;IAC5B,0EAA0E;IAC1E,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC;CAC7B;AAED;0FAC0F;AAC1F,MAAM,WAAW,sBAAsB;IACrC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,iBAAiB,EAAE,2BAA2B,CAAC;CACzD;AAED,wEAAwE;AACxE,wBAAgB,qBAAqB,CACnC,OAAO,EAAE,cAAc,GACtB,sBAAsB,CAYxB;AAED;;;;GAIG;AACH,wBAAgB,sBAAsB,CACpC,UAAU,EAAE,UAAU,EACtB,QAAQ,EAAE,SAAS,MAAM,EAAE,GAC1B,IAAI,CAWN;AAED,4EAA4E;AAC5E,wBAAuB,cAAc,CAAC,SAAS,EAC7C,SAAS,EAAE,aAAa,CAAC,SAAS,CAAC,EACnC,KAAK,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,GACtC,aAAa,CAAC,SAAS,GAAG,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAInD;AAED,mEAAmE;AACnE,wBAAsB,cAAc,CAClC,MAAM,EAAE,MAAM,EACd,UAAU,EAAE,MAAM,EAClB,QAAQ,EAAE,MAAM,GACf,OAAO,CAAC,IAAI,CAAC,CAKf"}
|
package/dist/rebuild-support.js
CHANGED
|
@@ -2,8 +2,13 @@ import { DEFAULT_LOCK_TTL_MS } from './lock.js';
|
|
|
2
2
|
import { DEFAULT_BATCH_SIZE } from './import.js';
|
|
3
3
|
/** Apply the shared defaults, once, so neither writer restates them. */
|
|
4
4
|
export function resolveRebuildOptions(options) {
|
|
5
|
-
const { batchSize = DEFAULT_BATCH_SIZE, lockTtlMs = DEFAULT_LOCK_TTL_MS, ...
|
|
6
|
-
return {
|
|
5
|
+
const { batchSize = DEFAULT_BATCH_SIZE, lockTtlMs = DEFAULT_LOCK_TTL_MS, ...definitionOptions } = options;
|
|
6
|
+
return {
|
|
7
|
+
name: definitionOptions.name,
|
|
8
|
+
batchSize,
|
|
9
|
+
lockTtlMs,
|
|
10
|
+
definitionOptions,
|
|
11
|
+
};
|
|
7
12
|
}
|
|
8
13
|
/**
|
|
9
14
|
* Reject a {@link SearchType} that declares any of the bookkeeping field names
|
package/dist/search.d.ts
CHANGED
|
@@ -1,20 +1,20 @@
|
|
|
1
1
|
import type { Client } from 'typesense';
|
|
2
2
|
import { type LocalizedValue, type SearchEngine, type SearchResult, type SearchSchema, type SearchType, type TextField } from '@lde/search';
|
|
3
3
|
import { type BuildSearchParamsOptions } from './query-compiler.js';
|
|
4
|
-
/** Where the engine reads documents
|
|
4
|
+
/** Where the engine reads documents – plus every query-compiler knob
|
|
5
5
|
* ({@link BuildSearchParamsOptions}), declared once there and forwarded
|
|
6
6
|
* wholesale into each search. `TypeName` is the union of the schema’s type
|
|
7
7
|
* names, so a literal schema makes the `collections` map exhaustive at
|
|
8
8
|
* compile time. Reference labels resolve per field from the collection of
|
|
9
|
-
* the SearchType its `labelSource` names
|
|
9
|
+
* the SearchType its `labelSource` names – a label source is just another
|
|
10
10
|
* entry in `collections`. */
|
|
11
11
|
export interface TypesenseSearchEngineOptions<TypeName extends string = string> extends BuildSearchParamsOptions {
|
|
12
|
-
/** The collection (or alias) per search type, keyed by the type’s `name`
|
|
12
|
+
/** The collection (or alias) per search type, keyed by the type’s `name` –
|
|
13
13
|
* with a literal schema, omitting a type is a compile error. */
|
|
14
14
|
readonly collections: Readonly<Record<TypeName, string>>;
|
|
15
15
|
/**
|
|
16
16
|
* Called when reference-label resolution fails; the search then degrades to
|
|
17
|
-
* id-only references rather than failing. Optional
|
|
17
|
+
* id-only references rather than failing. Optional – omit to swallow silently.
|
|
18
18
|
*/
|
|
19
19
|
readonly onLabelError?: (error: unknown) => void;
|
|
20
20
|
/**
|
|
@@ -40,12 +40,12 @@ export interface LabelSource {
|
|
|
40
40
|
}
|
|
41
41
|
/**
|
|
42
42
|
* A Typesense-backed {@link SearchEngine}, bound to the whole
|
|
43
|
-
* {@link SearchSchema} at construction
|
|
43
|
+
* {@link SearchSchema} at construction – like every other schema consumer.
|
|
44
44
|
* Each type’s collection comes from `options.collections`. `search` compiles
|
|
45
45
|
* the query ({@link buildSearchParams}), runs it against the type’s
|
|
46
|
-
* collection, resolves the reference labels for the page of hits
|
|
46
|
+
* collection, resolves the reference labels for the page of hits – each
|
|
47
47
|
* reference field from its own label source’s collection, all sources bundled
|
|
48
|
-
* into one lookup
|
|
48
|
+
* into one lookup – and reconstructs the engine-neutral
|
|
49
49
|
* {@link SearchResult} ({@link parseSearchResponse}). `searchFacets` answers
|
|
50
50
|
* a whole facet batch (e.g. a faceted listing’s skip-own-filter query variants) as a
|
|
51
51
|
* single `multi_search` round-trip with one shared label lookup; a failed
|
|
@@ -55,7 +55,7 @@ export interface LabelSource {
|
|
|
55
55
|
*
|
|
56
56
|
* With a schema built by `searchSchema` over `defineSearchType` literals,
|
|
57
57
|
* `search()` accepts only the deployment’s own types and returns typo-safe
|
|
58
|
-
* facet/document keys per call
|
|
58
|
+
* facet/document keys per call – no caller-side generics.
|
|
59
59
|
*/
|
|
60
60
|
export declare function createTypesenseSearchEngine<const Types extends readonly SearchType[]>(client: Client, schema: SearchSchema<Types>, options: TypesenseSearchEngineOptions<Types[number]['name']>): SearchEngine<Types>;
|
|
61
61
|
/** One label lookup: a source and the distinct IRIs to resolve against it. */
|
|
@@ -64,13 +64,14 @@ export interface LabelLookupGroup {
|
|
|
64
64
|
readonly iris: readonly string[];
|
|
65
65
|
}
|
|
66
66
|
/**
|
|
67
|
-
* Resolve labels from each group’s label-source collection.
|
|
68
|
-
*
|
|
69
|
-
*
|
|
67
|
+
* Resolve labels from each group’s label-source collection. Labels are
|
|
68
|
+
* reconstructed from the source type’s `label` declaration ({@link
|
|
69
|
+
* localizedValue}), carrying every present language – including ones outside
|
|
70
|
+
* the declared `locales` and untagged (`und`) values.
|
|
70
71
|
*
|
|
71
72
|
* All groups travel as ONE `multi_search` (POST) call, each group’s id-list
|
|
72
73
|
* split over per-search batches: the id-list of a page or facet carrying many
|
|
73
|
-
* references
|
|
74
|
+
* references – e.g. a dataset with dozens of classes – would overflow
|
|
74
75
|
* Typesense’s GET query-string limit (4000 chars, and IRIs URL-encode to
|
|
75
76
|
* several times their length) if it travelled in the URL. POST puts it in the
|
|
76
77
|
* body; each batch stays under Typesense’s `per_page` cap, and bundling the
|
package/dist/search.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"search.d.ts","sourceRoot":"","sources":["../src/search.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,WAAW,CAAC;AACxC,OAAO,EAGL,KAAK,cAAc,EAMnB,KAAK,YAAY,EAEjB,KAAK,YAAY,EACjB,KAAK,YAAY,EACjB,KAAK,UAAU,EAEf,KAAK,SAAS,EACf,MAAM,aAAa,CAAC;
|
|
1
|
+
{"version":3,"file":"search.d.ts","sourceRoot":"","sources":["../src/search.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,WAAW,CAAC;AACxC,OAAO,EAGL,KAAK,cAAc,EAMnB,KAAK,YAAY,EAEjB,KAAK,YAAY,EACjB,KAAK,YAAY,EACjB,KAAK,UAAU,EAEf,KAAK,SAAS,EACf,MAAM,aAAa,CAAC;AAYrB,OAAO,EAGL,KAAK,wBAAwB,EAC9B,MAAM,qBAAqB,CAAC;AAE7B;;;;;;8BAM8B;AAC9B,MAAM,WAAW,4BAA4B,CAC3C,QAAQ,SAAS,MAAM,GAAG,MAAM,CAChC,SAAQ,wBAAwB;IAChC;qEACiE;IACjE,QAAQ,CAAC,WAAW,EAAE,QAAQ,CAAC,MAAM,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC,CAAC;IACzD;;;OAGG;IACH,QAAQ,CAAC,YAAY,CAAC,EAAE,CAAC,KAAK,EAAE,OAAO,KAAK,IAAI,CAAC;IACjD;;;;;;;OAOG;IACH,QAAQ,CAAC,eAAe,CAAC,EAAE,MAAM,CAAC;CACnC;AAED;;;;;GAKG;AACH,MAAM,WAAW,WAAW;IAC1B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,UAAU,EAAE,SAAS,CAAC;IAC/B,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;CAC1B;AAED;;;;;;;;;;;;;;;;;;GAkBG;AACH,wBAAgB,2BAA2B,CACzC,KAAK,CAAC,KAAK,SAAS,SAAS,UAAU,EAAE,EAEzC,MAAM,EAAE,MAAM,EACd,MAAM,EAAE,YAAY,CAAC,KAAK,CAAC,EAC3B,OAAO,EAAE,4BAA4B,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,CAAC,GAC3D,YAAY,CAAC,KAAK,CAAC,CAsRrB;AAED,8EAA8E;AAC9E,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,CAAC,MAAM,EAAE,WAAW,CAAC;IAC7B,QAAQ,CAAC,IAAI,EAAE,SAAS,MAAM,EAAE,CAAC;CAClC;AA0FD;;;;;;;;;;;;;;GAcG;AACH,wBAAsB,WAAW,CAC/B,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,aAAa,CAAC,EACnC,MAAM,EAAE,SAAS,gBAAgB,EAAE,EACnC,OAAO,CAAC,EAAE,CAAC,KAAK,EAAE,KAAK,KAAK,IAAI,GAC/B,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,cAAc,CAAC,CAAC,CAgDtC;AA6BD,oEAAoE;AACpE,MAAM,WAAW,uBAAuB;IACtC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,IAAI,CAAC,EAAE,SAAS;QAAE,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;KAAE,EAAE,CAAC;IAC1E,QAAQ,CAAC,YAAY,CAAC,EAAE,SAAS;QAC/B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;QAC5B,QAAQ,CAAC,MAAM,EAAE,SAAS;YACxB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;YACvB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;SACxB,EAAE,CAAC;KACL,EAAE,CAAC;CACL;AAED;;;;;;GAMG;AACH,wBAAgB,mBAAmB,CACjC,QAAQ,EAAE,uBAAuB,EACjC,UAAU,EAAE,UAAU,EACtB,MAAM,EAAE,WAAW,CAAC,MAAM,EAAE,cAAc,CAAC,GAC1C,YAAY,CAsCd"}
|
package/dist/search.js
CHANGED
|
@@ -1,13 +1,13 @@
|
|
|
1
|
-
import { assertTypeInSchema, assertValidQuery, fieldNamed, isRangeFacet, labelFieldOf, outputFields, physicalFields, referenceFields, } from '@lde/search/adapter';
|
|
1
|
+
import { assertTypeInSchema, assertValidQuery, displayLangOf, fieldNamed, isRangeFacet, labelFieldOf, outputFields, physicalFields, referenceFields, } from '@lde/search/adapter';
|
|
2
2
|
import { buildSearchParams, escapeFilterValue, } from './query-compiler.js';
|
|
3
3
|
/**
|
|
4
4
|
* A Typesense-backed {@link SearchEngine}, bound to the whole
|
|
5
|
-
* {@link SearchSchema} at construction
|
|
5
|
+
* {@link SearchSchema} at construction – like every other schema consumer.
|
|
6
6
|
* Each type’s collection comes from `options.collections`. `search` compiles
|
|
7
7
|
* the query ({@link buildSearchParams}), runs it against the type’s
|
|
8
|
-
* collection, resolves the reference labels for the page of hits
|
|
8
|
+
* collection, resolves the reference labels for the page of hits – each
|
|
9
9
|
* reference field from its own label source’s collection, all sources bundled
|
|
10
|
-
* into one lookup
|
|
10
|
+
* into one lookup – and reconstructs the engine-neutral
|
|
11
11
|
* {@link SearchResult} ({@link parseSearchResponse}). `searchFacets` answers
|
|
12
12
|
* a whole facet batch (e.g. a faceted listing’s skip-own-filter query variants) as a
|
|
13
13
|
* single `multi_search` round-trip with one shared label lookup; a failed
|
|
@@ -17,25 +17,25 @@ import { buildSearchParams, escapeFilterValue, } from './query-compiler.js';
|
|
|
17
17
|
*
|
|
18
18
|
* With a schema built by `searchSchema` over `defineSearchType` literals,
|
|
19
19
|
* `search()` accepts only the deployment’s own types and returns typo-safe
|
|
20
|
-
* facet/document keys per call
|
|
20
|
+
* facet/document keys per call – no caller-side generics.
|
|
21
21
|
*/
|
|
22
22
|
export function createTypesenseSearchEngine(client, schema, options) {
|
|
23
23
|
// Resolve every type's collection ONCE at construction, so a
|
|
24
|
-
// misconfiguration fails at startup
|
|
24
|
+
// misconfiguration fails at startup – never on the first search that
|
|
25
25
|
// happens to hit the unwired type.
|
|
26
26
|
const collections = new Map([...schema.values()].map((searchType) => {
|
|
27
27
|
const collection = options.collections[searchType.name];
|
|
28
28
|
if (collection === undefined) {
|
|
29
29
|
throw new Error(`No collection configured for search type “${searchType.name}”; set options.collections.${searchType.name}.`);
|
|
30
30
|
}
|
|
31
|
-
return [searchType.
|
|
31
|
+
return [searchType.class, collection];
|
|
32
32
|
}));
|
|
33
33
|
// Resolve every reference field's label source ONCE at construction. The
|
|
34
34
|
// schema already guarantees the named type exists and serves labels
|
|
35
35
|
// (`searchSchema`/`labelFieldOf`), and `collections` is exhaustive per type.
|
|
36
36
|
const typesByName = new Map([...schema.values()].map((searchType) => [searchType.name, searchType]));
|
|
37
37
|
const labelSources = new Map([...schema.values()].map((searchType) => [
|
|
38
|
-
searchType.
|
|
38
|
+
searchType.class,
|
|
39
39
|
new Map(referenceFields(searchType)
|
|
40
40
|
.filter((field) => field.labelSource !== undefined)
|
|
41
41
|
.map((field) => {
|
|
@@ -44,7 +44,7 @@ export function createTypesenseSearchEngine(client, schema, options) {
|
|
|
44
44
|
return [
|
|
45
45
|
field.name,
|
|
46
46
|
{
|
|
47
|
-
collection: collections.get(source.
|
|
47
|
+
collection: collections.get(source.class),
|
|
48
48
|
labelField,
|
|
49
49
|
queryBy: physicalFields(labelField).search.join(','),
|
|
50
50
|
},
|
|
@@ -63,9 +63,9 @@ export function createTypesenseSearchEngine(client, schema, options) {
|
|
|
63
63
|
// resolved label source – fixed at construction, so labelLookupGroups need
|
|
64
64
|
// not re-derive or re-resolve them on every search.
|
|
65
65
|
const outputReferenceSources = new Map([...schema.values()].map((searchType) => {
|
|
66
|
-
const sources = labelSources.get(searchType.
|
|
66
|
+
const sources = labelSources.get(searchType.class);
|
|
67
67
|
return [
|
|
68
|
-
searchType.
|
|
68
|
+
searchType.class,
|
|
69
69
|
referenceFields(searchType)
|
|
70
70
|
.filter((field) => field.output === true && sources?.has(field.name))
|
|
71
71
|
.map((field) => ({
|
|
@@ -129,14 +129,14 @@ export function createTypesenseSearchEngine(client, schema, options) {
|
|
|
129
129
|
// cannot leave an unhandled rejection behind if the search itself fails.
|
|
130
130
|
// `undefined` when the cache is off or the type has no label sources.
|
|
131
131
|
function startCachedLabels(searchType) {
|
|
132
|
-
const sources = distinctLabelSources.get(searchType.
|
|
132
|
+
const sources = distinctLabelSources.get(searchType.class);
|
|
133
133
|
if (options.labelCacheTtlMs === undefined ||
|
|
134
134
|
sources === undefined ||
|
|
135
135
|
sources.length === 0) {
|
|
136
136
|
return undefined;
|
|
137
137
|
}
|
|
138
138
|
const ttlMs = options.labelCacheTtlMs;
|
|
139
|
-
return Promise.all(sources.map((source) => cachedAllLabels(source, ttlMs))).then((maps) => mergeCachedLabels(searchType.
|
|
139
|
+
return Promise.all(sources.map((source) => cachedAllLabels(source, ttlMs))).then((maps) => mergeCachedLabels(searchType.class, maps));
|
|
140
140
|
}
|
|
141
141
|
// Labels are supplementary: a failed lookup (e.g. a label-source collection
|
|
142
142
|
// mid-rebuild) degrades to id-only references rather than failing the whole
|
|
@@ -165,10 +165,10 @@ export function createTypesenseSearchEngine(client, schema, options) {
|
|
|
165
165
|
const params = buildSearchParams(query, searchType, options);
|
|
166
166
|
const cachedLabelsPromise = startCachedLabels(searchType);
|
|
167
167
|
const response = (await client
|
|
168
|
-
.collections(collections.get(searchType.
|
|
168
|
+
.collections(collections.get(searchType.class))
|
|
169
169
|
.documents()
|
|
170
170
|
.search(params));
|
|
171
|
-
const labels = await resolveLabels(cachedLabelsPromise, () => labelLookupGroups([response], labelSources.get(searchType.
|
|
171
|
+
const labels = await resolveLabels(cachedLabelsPromise, () => labelLookupGroups([response], labelSources.get(searchType.class), outputReferenceSources.get(searchType.class) ?? []));
|
|
172
172
|
return parseSearchResponse(response, searchType, labels);
|
|
173
173
|
},
|
|
174
174
|
async searchFacets(searchType, queries) {
|
|
@@ -179,7 +179,7 @@ export function createTypesenseSearchEngine(client, schema, options) {
|
|
|
179
179
|
if (queries.length === 0) {
|
|
180
180
|
return [];
|
|
181
181
|
}
|
|
182
|
-
const collection = collections.get(searchType.
|
|
182
|
+
const collection = collections.get(searchType.class);
|
|
183
183
|
const cachedLabelsPromise = startCachedLabels(searchType);
|
|
184
184
|
// The whole batch travels as ONE multi_search round-trip. Each query
|
|
185
185
|
// compiles as facet-only regardless of what it carries: no hits
|
|
@@ -193,7 +193,7 @@ export function createTypesenseSearchEngine(client, schema, options) {
|
|
|
193
193
|
}));
|
|
194
194
|
// One label lookup serves every successful facet result in the batch.
|
|
195
195
|
const responses = results.filter((result) => !('error' in result));
|
|
196
|
-
const labels = await resolveLabels(cachedLabelsPromise, () => labelLookupGroups(responses, labelSources.get(searchType.
|
|
196
|
+
const labels = await resolveLabels(cachedLabelsPromise, () => labelLookupGroups(responses, labelSources.get(searchType.class), outputReferenceSources.get(searchType.class) ?? []));
|
|
197
197
|
// multi_search reports a failed entry inline instead of rejecting the
|
|
198
198
|
// call; pass that through as a per-query outcome – naming the facets,
|
|
199
199
|
// not the position, since the batch order is the caller's internal –
|
|
@@ -211,7 +211,7 @@ export function createTypesenseSearchEngine(client, schema, options) {
|
|
|
211
211
|
/**
|
|
212
212
|
* Load a FULL label-source collection into a label map via the documents
|
|
213
213
|
* export endpoint, which streams every document as JSONL (one JSON object per
|
|
214
|
-
* line). Each line is reconstructed by {@link
|
|
214
|
+
* line). Each line is reconstructed by {@link localizedValue}, exactly as the
|
|
215
215
|
* per-search {@link fetchLabels} path does for its `multi_search` hits.
|
|
216
216
|
*/
|
|
217
217
|
async function loadAllLabels(client, source) {
|
|
@@ -225,7 +225,7 @@ async function loadAllLabels(client, source) {
|
|
|
225
225
|
continue;
|
|
226
226
|
}
|
|
227
227
|
const document = JSON.parse(line);
|
|
228
|
-
const label =
|
|
228
|
+
const label = localizedValue(document, source.labelField);
|
|
229
229
|
if (label !== undefined) {
|
|
230
230
|
labels.set(String(document.id), label);
|
|
231
231
|
}
|
|
@@ -286,13 +286,14 @@ function labelLookupGroups(responses, sources, outputSources) {
|
|
|
286
286
|
}));
|
|
287
287
|
}
|
|
288
288
|
/**
|
|
289
|
-
* Resolve labels from each group’s label-source collection.
|
|
290
|
-
*
|
|
291
|
-
*
|
|
289
|
+
* Resolve labels from each group’s label-source collection. Labels are
|
|
290
|
+
* reconstructed from the source type’s `label` declaration ({@link
|
|
291
|
+
* localizedValue}), carrying every present language – including ones outside
|
|
292
|
+
* the declared `locales` and untagged (`und`) values.
|
|
292
293
|
*
|
|
293
294
|
* All groups travel as ONE `multi_search` (POST) call, each group’s id-list
|
|
294
295
|
* split over per-search batches: the id-list of a page or facet carrying many
|
|
295
|
-
* references
|
|
296
|
+
* references – e.g. a dataset with dozens of classes – would overflow
|
|
296
297
|
* Typesense’s GET query-string limit (4000 chars, and IRIs URL-encode to
|
|
297
298
|
* several times their length) if it travelled in the URL. POST puts it in the
|
|
298
299
|
* body; each batch stays under Typesense’s `per_page` cap, and bundling the
|
|
@@ -331,7 +332,7 @@ export async function fetchLabels(client, groups, onError) {
|
|
|
331
332
|
return;
|
|
332
333
|
}
|
|
333
334
|
for (const hit of result.hits ?? []) {
|
|
334
|
-
const label =
|
|
335
|
+
const label = localizedValue(hit.document, groupPerSearch[index].source.labelField);
|
|
335
336
|
if (label !== undefined) {
|
|
336
337
|
labels.set(String(hit.document.id), label);
|
|
337
338
|
}
|
|
@@ -342,21 +343,6 @@ export async function fetchLabels(client, groups, onError) {
|
|
|
342
343
|
/** Typesense caps `per_page` at 250; the multi_search POST body holds the
|
|
343
344
|
* id-list comfortably, so resolve references in batches of this size. */
|
|
344
345
|
const LABEL_BATCH_SIZE = 200;
|
|
345
|
-
/**
|
|
346
|
-
* Reconstruct a label document into a language map via the source type’s
|
|
347
|
-
* `label` declaration (its per-locale display fields), with a bare untagged
|
|
348
|
-
* `label` value as the `und` fallback when no locale variant exists, or
|
|
349
|
-
* `undefined` when the document carries no usable label – so the reference
|
|
350
|
-
* stays id-only rather than gaining an empty label.
|
|
351
|
-
*/
|
|
352
|
-
function labelValue(document, labelField) {
|
|
353
|
-
const localized = localizedValue(document, labelField);
|
|
354
|
-
if (localized !== undefined) {
|
|
355
|
-
return localized;
|
|
356
|
-
}
|
|
357
|
-
const untagged = document[labelField.name];
|
|
358
|
-
return typeof untagged === 'string' ? { und: [untagged] } : undefined;
|
|
359
|
-
}
|
|
360
346
|
/** Merge per-collection label maps into one IRI-keyed map (URI identity). */
|
|
361
347
|
function mergeLabels(maps) {
|
|
362
348
|
if (maps.length === 1) {
|
|
@@ -383,7 +369,7 @@ export function parseSearchResponse(response, searchType, labels) {
|
|
|
383
369
|
document: reconstructDocument(hit.document, searchType, labels),
|
|
384
370
|
}));
|
|
385
371
|
// Reference facets are IRI-keyed; their buckets carry a resolved data label.
|
|
386
|
-
// Plain facets (tokens, free strings) carry no label
|
|
372
|
+
// Plain facets (tokens, free strings) carry no label – the consumer owns display.
|
|
387
373
|
// Only reference facets with a label source get bucket labels; an id-only
|
|
388
374
|
// reference facet stays id-only even when a cached full-collection map holds
|
|
389
375
|
// its IRIs.
|
|
@@ -447,16 +433,24 @@ function logicalValue(flat, field, labels) {
|
|
|
447
433
|
return flat[field.name] === true;
|
|
448
434
|
}
|
|
449
435
|
}
|
|
450
|
-
/**
|
|
436
|
+
/**
|
|
437
|
+
* Gather every present-language display field back into a language map. Display
|
|
438
|
+
* fields are pattern-based (`${name}_<lang>`, {@link displayLangOf}), so this
|
|
439
|
+
* recovers languages outside the declared `locales` too – a value tagged in an
|
|
440
|
+
* undeclared language, or untagged (`und`), still reconstructs rather than being
|
|
441
|
+
* dropped.
|
|
442
|
+
*/
|
|
451
443
|
function localizedValue(flat, field) {
|
|
452
444
|
const map = {};
|
|
453
|
-
const
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
if (typeof value === 'string') {
|
|
457
|
-
map[locale] = [value];
|
|
445
|
+
for (const [key, value] of Object.entries(flat)) {
|
|
446
|
+
if (typeof value !== 'string') {
|
|
447
|
+
continue;
|
|
458
448
|
}
|
|
459
|
-
|
|
449
|
+
const lang = displayLangOf(field, key);
|
|
450
|
+
if (lang !== undefined) {
|
|
451
|
+
map[lang] = [value];
|
|
452
|
+
}
|
|
453
|
+
}
|
|
460
454
|
return Object.keys(map).length > 0 ? map : undefined;
|
|
461
455
|
}
|
|
462
456
|
/** Map stored reference IRIs to labelled references; id-only when no label. */
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@lde/search-typesense",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.9.0",
|
|
4
4
|
"description": "Typesense implementation of the @lde/search SearchEngine port: collection-schema builder, query compiler, label-resolving result reconstruction, and blue/green index lifecycle. Engine-specific (Typesense) but domain-agnostic.",
|
|
5
5
|
"repository": {
|
|
6
6
|
"url": "git+https://github.com/ldelements/lde.git",
|
|
@@ -25,7 +25,7 @@
|
|
|
25
25
|
"!**/*.tsbuildinfo"
|
|
26
26
|
],
|
|
27
27
|
"dependencies": {
|
|
28
|
-
"@lde/search": "^0.
|
|
28
|
+
"@lde/search": "^0.7.0",
|
|
29
29
|
"@lde/text-normalization": "^0.1.1",
|
|
30
30
|
"tslib": "^2.3.0",
|
|
31
31
|
"typesense": "^3.0.6"
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"collection-schema.d.ts","sourceRoot":"","sources":["../src/collection-schema.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,WAAW,CAAC;AAExD,OAAO,EAAoB,KAAK,UAAU,EAAE,MAAM,aAAa,CAAC;AAGhE,0EAA0E;AAC1E,MAAM,WAAW,uBAAuB;IACtC,gDAAgD;IAChD,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB;;;4BAGwB;IACxB,QAAQ,CAAC,aAAa,CAAC,EAAE,MAAM,CAAC;IAChC,kEAAkE;IAClE,QAAQ,CAAC,mBAAmB,CAAC,EAAE,MAAM,CAAC;IACtC,kEAAkE;IAClE,QAAQ,CAAC,WAAW,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;CAC1C;AAED;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,wBAAgB,qBAAqB,CACnC,UAAU,EAAE,UAAU,EACtB,OAAO,EAAE,uBAAuB,GAC/B,sBAAsB,CAexB"}
|