@etohq/search 0.0.0-stage → 2.0.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/NOTICE +24 -0
- package/README.md +188 -2
- package/dist/index.d.ts +12 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +14 -0
- package/dist/index.js.map +1 -0
- package/dist/joiner-config.d.ts +8 -0
- package/dist/joiner-config.d.ts.map +1 -0
- package/dist/joiner-config.js +12 -0
- package/dist/joiner-config.js.map +1 -0
- package/dist/loaders/providers.d.ts +5 -0
- package/dist/loaders/providers.d.ts.map +1 -0
- package/dist/loaders/providers.js +23 -0
- package/dist/loaders/providers.js.map +1 -0
- package/dist/migrations/Migration20260805085626.d.ts +6 -0
- package/dist/migrations/Migration20260805085626.d.ts.map +1 -0
- package/dist/migrations/Migration20260805085626.js +23 -0
- package/dist/migrations/Migration20260805085626.js.map +1 -0
- package/dist/migrations/Migration20260903085616.d.ts +6 -0
- package/dist/migrations/Migration20260903085616.d.ts.map +1 -0
- package/dist/migrations/Migration20260903085616.js +32 -0
- package/dist/migrations/Migration20260903085616.js.map +1 -0
- package/dist/migrations/Migration20260911115931.d.ts +6 -0
- package/dist/migrations/Migration20260911115931.d.ts.map +1 -0
- package/dist/migrations/Migration20260911115931.js +14 -0
- package/dist/migrations/Migration20260911115931.js.map +1 -0
- package/dist/models/index.d.ts +4 -0
- package/dist/models/index.d.ts.map +1 -0
- package/dist/models/index.js +10 -0
- package/dist/models/index.js.map +1 -0
- package/dist/models/search-index-sync.d.ts +34 -0
- package/dist/models/search-index-sync.d.ts.map +1 -0
- package/dist/models/search-index-sync.js +47 -0
- package/dist/models/search-index-sync.js.map +1 -0
- package/dist/models/search-index-version.d.ts +34 -0
- package/dist/models/search-index-version.d.ts.map +1 -0
- package/dist/models/search-index-version.js +37 -0
- package/dist/models/search-index-version.js.map +1 -0
- package/dist/models/search-index.d.ts +28 -0
- package/dist/models/search-index.d.ts.map +1 -0
- package/dist/models/search-index.js +28 -0
- package/dist/models/search-index.js.map +1 -0
- package/dist/services/index.d.ts +3 -0
- package/dist/services/index.d.ts.map +1 -0
- package/dist/services/index.js +11 -0
- package/dist/services/index.js.map +1 -0
- package/dist/services/search-module-service.d.ts +72 -0
- package/dist/services/search-module-service.d.ts.map +1 -0
- package/dist/services/search-module-service.js +336 -0
- package/dist/services/search-module-service.js.map +1 -0
- package/dist/services/search-provider.d.ts +23 -0
- package/dist/services/search-provider.d.ts.map +1 -0
- package/dist/services/search-provider.js +64 -0
- package/dist/services/search-provider.js.map +1 -0
- package/dist/tsconfig.tsbuildinfo +1 -0
- package/dist/types/index.d.ts +131 -0
- package/dist/types/index.d.ts.map +1 -0
- package/dist/types/index.js +7 -0
- package/dist/types/index.js.map +1 -0
- package/dist/utils/active-version-cache.d.ts +36 -0
- package/dist/utils/active-version-cache.d.ts.map +1 -0
- package/dist/utils/active-version-cache.js +73 -0
- package/dist/utils/active-version-cache.js.map +1 -0
- package/dist/utils/deletion.d.ts +11 -0
- package/dist/utils/deletion.d.ts.map +1 -0
- package/dist/utils/deletion.js +50 -0
- package/dist/utils/deletion.js.map +1 -0
- package/dist/utils/index.d.ts +88 -0
- package/dist/utils/index.d.ts.map +1 -0
- package/dist/utils/index.js +524 -0
- package/dist/utils/index.js.map +1 -0
- package/dist/utils/ingestion.d.ts +12 -0
- package/dist/utils/ingestion.d.ts.map +1 -0
- package/dist/utils/ingestion.js +78 -0
- package/dist/utils/ingestion.js.map +1 -0
- package/dist/utils/migrations.d.ts +5 -0
- package/dist/utils/migrations.d.ts.map +1 -0
- package/dist/utils/migrations.js +143 -0
- package/dist/utils/migrations.js.map +1 -0
- package/dist/utils/rate-limit.d.ts +9 -0
- package/dist/utils/rate-limit.d.ts.map +1 -0
- package/dist/utils/rate-limit.js +47 -0
- package/dist/utils/rate-limit.js.map +1 -0
- package/dist/utils/seeding.d.ts +29 -0
- package/dist/utils/seeding.d.ts.map +1 -0
- package/dist/utils/seeding.js +639 -0
- package/dist/utils/seeding.js.map +1 -0
- package/dist/utils/versions.d.ts +50 -0
- package/dist/utils/versions.d.ts.map +1 -0
- package/dist/utils/versions.js +113 -0
- package/dist/utils/versions.js.map +1 -0
- package/package.json +56 -3
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"search-provider.d.ts","sourceRoot":"","sources":["../../src/services/search-provider.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,WAAW,EAAE,MAAM,wBAAwB,CAAA;AAEjE,OAAO,EAAE,gCAAgC,EAAE,MAAM,QAAQ,CAAA;AAEzD,KAAK,oBAAoB,GAAG;IAC1B,CACE,GAAG,EAAE,GAAG,OAAO,gCAAgC,GAAG,MAAM,EAAE,GACzD,WAAW,CAAC,eAAe,CAAA;CAC/B,CAAA;AAED;;;GAGG;AACH,qBAAa,qBAAqB;IAChC,SAAS,CAAC,QAAQ,CAAC,UAAU,EAAE,GAAG,CAAC,MAAM,EAAE,WAAW,CAAC,eAAe,CAAC,CAC5D;gBAEC,SAAS,EAAE,oBAAoB;IAyB3C,MAAM,CAAC,yBAAyB,CAC9B,aAAa,EAAE,WAAW,CAAC,WAAW,CAAC,eAAe,CAAC,EACvD,UAAU,CAAC,EAAE,MAAM,GAClB,MAAM;IAIT,QAAQ,CAAC,UAAU,EAAE,MAAM,GAAG,WAAW,CAAC,eAAe;IAezD,IAAI,IAAI,WAAW,CAAC,eAAe,EAAE;IAIrC;;;OAGG;IACH,oBAAoB,CAAC,UAAU,CAAC,EAAE,MAAM,GAAG,MAAM;CA2BlD"}
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.SearchProviderService = void 0;
|
|
4
|
+
const utils_1 = require("@etohq/framework/utils");
|
|
5
|
+
const _types_1 = require("../types");
|
|
6
|
+
/**
|
|
7
|
+
* Registry over the registered providers. Multiple providers can be used, and
|
|
8
|
+
* each index can select which provider to use for itself.
|
|
9
|
+
*/
|
|
10
|
+
class SearchProviderService {
|
|
11
|
+
constructor(container) {
|
|
12
|
+
this.providers_ = new Map();
|
|
13
|
+
for (const key of Object.keys(container)) {
|
|
14
|
+
if (!key.startsWith(_types_1.SearchProviderRegistrationPrefix)) {
|
|
15
|
+
continue;
|
|
16
|
+
}
|
|
17
|
+
const provider = container[key];
|
|
18
|
+
if (!provider?.identifier) {
|
|
19
|
+
continue;
|
|
20
|
+
}
|
|
21
|
+
// Definitions bind to a provider by identifier, so two registrations of
|
|
22
|
+
// the same provider class would be indistinguishable to them — the second
|
|
23
|
+
// would silently shadow the first.
|
|
24
|
+
if (this.providers_.has(provider.identifier)) {
|
|
25
|
+
throw new utils_1.EtoError(utils_1.EtoError.Types.INVALID_DATA, `The search provider "${provider.identifier}" is registered twice. Register each provider once; index definitions select it through their "provider" field.`);
|
|
26
|
+
}
|
|
27
|
+
this.providers_.set(provider.identifier, provider);
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
static getRegistrationIdentifier(providerClass, optionName) {
|
|
31
|
+
return `${providerClass.identifier}_${optionName}`;
|
|
32
|
+
}
|
|
33
|
+
retrieve(identifier) {
|
|
34
|
+
const provider = this.providers_.get(identifier);
|
|
35
|
+
if (!provider) {
|
|
36
|
+
throw new utils_1.EtoError(utils_1.EtoError.Types.NOT_FOUND, `Search provider "${identifier}" is not registered. Registered providers: ${[...this.providers_.keys()].join(", ") || "(none)"}`);
|
|
37
|
+
}
|
|
38
|
+
return provider;
|
|
39
|
+
}
|
|
40
|
+
list() {
|
|
41
|
+
return [...this.providers_.values()];
|
|
42
|
+
}
|
|
43
|
+
/**
|
|
44
|
+
* Used by definitions that do not name a provider. Resolves to the sole
|
|
45
|
+
* registered provider, or to the module's `default_provider` option.
|
|
46
|
+
*/
|
|
47
|
+
getDefaultIdentifier(configured) {
|
|
48
|
+
if (configured) {
|
|
49
|
+
// Surfaces a typo in `default_provider` at boot rather than on first query.
|
|
50
|
+
this.retrieve(configured);
|
|
51
|
+
return configured;
|
|
52
|
+
}
|
|
53
|
+
const identifiers = [...this.providers_.keys()];
|
|
54
|
+
if (identifiers.length === 1) {
|
|
55
|
+
return identifiers[0];
|
|
56
|
+
}
|
|
57
|
+
if (!identifiers.length) {
|
|
58
|
+
throw new utils_1.EtoError(utils_1.EtoError.Types.INVALID_DATA, "The Search Module requires at least one provider to be registered");
|
|
59
|
+
}
|
|
60
|
+
throw new utils_1.EtoError(utils_1.EtoError.Types.INVALID_DATA, `Several search providers are registered (${identifiers.join(", ")}), so "default_provider" must be set, or every index definition must name its provider`);
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
exports.SearchProviderService = SearchProviderService;
|
|
64
|
+
//# sourceMappingURL=search-provider.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"search-provider.js","sourceRoot":"","sources":["../../src/services/search-provider.ts"],"names":[],"mappings":";;;AACA,kDAAiD;AACjD,mCAAyD;AAQzD;;;GAGG;AACH,MAAa,qBAAqB;IAIhC,YAAY,SAA+B;QAHxB,eAAU,GAC3B,IAAI,GAAG,EAAE,CAAA;QAGT,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC;YACzC,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,yCAAgC,CAAC,EAAE,CAAC;gBACtD,SAAQ;YACV,CAAC;YAED,MAAM,QAAQ,GAAG,SAAS,CAAC,GAAG,CAAC,CAAA;YAC/B,IAAI,CAAC,QAAQ,EAAE,UAAU,EAAE,CAAC;gBAC1B,SAAQ;YACV,CAAC;YAED,wEAAwE;YACxE,0EAA0E;YAC1E,mCAAmC;YACnC,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,QAAQ,CAAC,UAAU,CAAC,EAAE,CAAC;gBAC7C,MAAM,IAAI,gBAAQ,CAChB,gBAAQ,CAAC,KAAK,CAAC,YAAY,EAC3B,wBAAwB,QAAQ,CAAC,UAAU,iHAAiH,CAC7J,CAAA;YACH,CAAC;YAED,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,QAAQ,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAA;QACpD,CAAC;IACH,CAAC;IAED,MAAM,CAAC,yBAAyB,CAC9B,aAAuD,EACvD,UAAmB;QAEnB,OAAO,GAAI,aAAqB,CAAC,UAAU,IAAI,UAAU,EAAE,CAAA;IAC7D,CAAC;IAED,QAAQ,CAAC,UAAkB;QACzB,MAAM,QAAQ,GAAG,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,UAAU,CAAC,CAAA;QAEhD,IAAI,CAAC,QAAQ,EAAE,CAAC;YACd,MAAM,IAAI,gBAAQ,CAChB,gBAAQ,CAAC,KAAK,CAAC,SAAS,EACxB,oBAAoB,UAAU,8CAC5B,CAAC,GAAG,IAAI,CAAC,UAAU,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,QAC5C,EAAE,CACH,CAAA;QACH,CAAC;QAED,OAAO,QAAQ,CAAA;IACjB,CAAC;IAED,IAAI;QACF,OAAO,CAAC,GAAG,IAAI,CAAC,UAAU,CAAC,MAAM,EAAE,CAAC,CAAA;IACtC,CAAC;IAED;;;OAGG;IACH,oBAAoB,CAAC,UAAmB;QACtC,IAAI,UAAU,EAAE,CAAC;YACf,4EAA4E;YAC5E,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAA;YACzB,OAAO,UAAU,CAAA;QACnB,CAAC;QAED,MAAM,WAAW,GAAG,CAAC,GAAG,IAAI,CAAC,UAAU,CAAC,IAAI,EAAE,CAAC,CAAA;QAE/C,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC7B,OAAO,WAAW,CAAC,CAAC,CAAC,CAAA;QACvB,CAAC;QAED,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE,CAAC;YACxB,MAAM,IAAI,gBAAQ,CAChB,gBAAQ,CAAC,KAAK,CAAC,YAAY,EAC3B,mEAAmE,CACpE,CAAA;QACH,CAAC;QAED,MAAM,IAAI,gBAAQ,CAChB,gBAAQ,CAAC,KAAK,CAAC,YAAY,EAC3B,4CAA4C,WAAW,CAAC,IAAI,CAC1D,IAAI,CACL,wFAAwF,CAC1F,CAAA;IACH,CAAC;CACF;AAtFD,sDAsFC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"root":["../src/index.ts","../src/joiner-config.ts","../src/loaders/providers.ts","../src/migrations/migration20260805085626.ts","../src/migrations/migration20260903085616.ts","../src/migrations/migration20260911115931.ts","../src/models/index.ts","../src/models/search-index-sync.ts","../src/models/search-index-version.ts","../src/models/search-index.ts","../src/services/index.ts","../src/services/search-module-service.ts","../src/services/search-provider.ts","../src/services/__tests__/event-subscription.spec.ts","../src/types/index.ts","../src/utils/active-version-cache.ts","../src/utils/deletion.ts","../src/utils/index.ts","../src/utils/ingestion.ts","../src/utils/migrations.ts","../src/utils/rate-limit.ts","../src/utils/seeding.ts","../src/utils/versions.ts","../src/utils/__tests__/active-version.spec.ts","../src/utils/__tests__/migrations.spec.ts","../src/utils/__tests__/rate-limit.spec.ts","../src/utils/__tests__/resolve-index-definitions.spec.ts","../src/utils/__tests__/seeding.spec.ts","../src/utils/__tests__/vector-query.spec.ts","../src/utils/__tests__/versions.spec.ts"],"version":"5.8.3"}
|
|
@@ -0,0 +1,131 @@
|
|
|
1
|
+
import { Logger, ModuleProviderExports, ModuleServiceInitializeOptions, ModulesSdkTypes, SearchTypes } from "@etohq/framework/types";
|
|
2
|
+
export declare const SearchProviderIdentifiersRegistrationName = "search_providers_identifier";
|
|
3
|
+
export declare const SearchProviderRegistrationPrefix = "srh_";
|
|
4
|
+
export type SearchModuleOptions = Partial<ModuleServiceInitializeOptions> & {
|
|
5
|
+
/**
|
|
6
|
+
* More than one may be registered, unlike most provider-backed modules. A
|
|
7
|
+
* definition picks one by identifier through its `provider`, so products can
|
|
8
|
+
* live on a dedicated engine while quieter entities stay on Postgres.
|
|
9
|
+
*/
|
|
10
|
+
providers?: {
|
|
11
|
+
resolve: string | ModuleProviderExports;
|
|
12
|
+
id: string;
|
|
13
|
+
options?: Record<string, unknown>;
|
|
14
|
+
}[];
|
|
15
|
+
default_provider?: string;
|
|
16
|
+
index_prefix?: string;
|
|
17
|
+
/**
|
|
18
|
+
* The definitions this module manages. It does not discover them itself — the
|
|
19
|
+
* application loads them off the file system and passes them in, so there is one
|
|
20
|
+
* input rather than two that can disagree. Already normalized: DSL field
|
|
21
|
+
* schemas are compiled to plain definitions by `defineSearchIndex`, so the
|
|
22
|
+
* module never sees them.
|
|
23
|
+
*/
|
|
24
|
+
indexes?: SearchTypes.SearchIndexDefinition[];
|
|
25
|
+
reindex?: {
|
|
26
|
+
batch_size?: number;
|
|
27
|
+
};
|
|
28
|
+
};
|
|
29
|
+
declare module "@etohq/types" {
|
|
30
|
+
interface ModuleOptions {
|
|
31
|
+
"@etohq/search": SearchModuleOptions;
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
/**
|
|
35
|
+
* Handed to the migration and seeding helpers per call; they keep no state of
|
|
36
|
+
* their own. `indexes` is a live reference to the service's registry, not a copy,
|
|
37
|
+
* and `providers` is structural so `utils` need not depend on the service layer.
|
|
38
|
+
*/
|
|
39
|
+
/** Resolved definitions, keyed by index name. */
|
|
40
|
+
export type SearchIndexes = Map<string, SearchTypes.ResolvedSearchIndexDefinition>;
|
|
41
|
+
export type SearchIndexContext = {
|
|
42
|
+
container: SearchTypes.SearchContainer;
|
|
43
|
+
logger: Logger;
|
|
44
|
+
options: SearchModuleOptions;
|
|
45
|
+
indexes: SearchIndexes;
|
|
46
|
+
providers: {
|
|
47
|
+
retrieve(identifier: string): SearchTypes.ISearchProvider;
|
|
48
|
+
};
|
|
49
|
+
indexService: ModulesSdkTypes.IEtoInternalService<any>;
|
|
50
|
+
versionService: ModulesSdkTypes.IEtoInternalService<any>;
|
|
51
|
+
syncService: ModulesSdkTypes.IEtoInternalService<any>;
|
|
52
|
+
locking?: {
|
|
53
|
+
execute<T>(keys: string | string[], job: (signal?: AbortSignal) => Promise<T>, args?: {
|
|
54
|
+
timeout?: number;
|
|
55
|
+
expire?: number;
|
|
56
|
+
}): Promise<T>;
|
|
57
|
+
};
|
|
58
|
+
activeVersionCache?: {
|
|
59
|
+
get(name: string, options?: {
|
|
60
|
+
fresh?: boolean;
|
|
61
|
+
}): Promise<{
|
|
62
|
+
physical_name: string;
|
|
63
|
+
provider: string;
|
|
64
|
+
version: number;
|
|
65
|
+
}>;
|
|
66
|
+
set(name: string, value: {
|
|
67
|
+
physical_name: string;
|
|
68
|
+
provider: string;
|
|
69
|
+
version: number;
|
|
70
|
+
}): Promise<void>;
|
|
71
|
+
invalidate(): Promise<void>;
|
|
72
|
+
};
|
|
73
|
+
};
|
|
74
|
+
export type SearchIndexRegistry = Pick<SearchIndexContext, "indexes" | "indexService" | "versionService" | "providers" | "logger">;
|
|
75
|
+
export type SearchSeedRuntime = SearchIndexRegistry & Pick<SearchIndexContext, "syncService" | "container" | "options" | "activeVersionCache">;
|
|
76
|
+
export type SearchIngestionRuntime = SearchIndexRegistry & Pick<SearchIndexContext, "container" | "activeVersionCache">;
|
|
77
|
+
/** Event name to the indexes that declared it. */
|
|
78
|
+
export type SearchEventRoutes = Map<string, string[]>;
|
|
79
|
+
/** The logical index and which of its versions currently serves reads. */
|
|
80
|
+
export type SearchIndexRecord = {
|
|
81
|
+
id: string;
|
|
82
|
+
name: string;
|
|
83
|
+
active_version: number | null;
|
|
84
|
+
};
|
|
85
|
+
/** One physical index ever built for a logical index. */
|
|
86
|
+
export type SearchIndexVersionRecord = {
|
|
87
|
+
id: string;
|
|
88
|
+
search_index_id: string;
|
|
89
|
+
version: number;
|
|
90
|
+
provider: string;
|
|
91
|
+
physical_name: string;
|
|
92
|
+
definition_hash: string;
|
|
93
|
+
status: string;
|
|
94
|
+
};
|
|
95
|
+
/** One seed run against one index version. Append-only, so these are the history. */
|
|
96
|
+
export type SearchIndexSyncRecord = {
|
|
97
|
+
id: string;
|
|
98
|
+
search_index_version_id: string;
|
|
99
|
+
job_id: string | null;
|
|
100
|
+
status: string;
|
|
101
|
+
filters: Record<string, unknown> | null;
|
|
102
|
+
last_key: string | null;
|
|
103
|
+
/** Whether a later run may continue from `last_key`. */
|
|
104
|
+
resumable: boolean;
|
|
105
|
+
documents_synced: number;
|
|
106
|
+
started_at: Date | null;
|
|
107
|
+
completed_at: Date | null;
|
|
108
|
+
error: string | null;
|
|
109
|
+
created_at: Date;
|
|
110
|
+
updated_at: Date;
|
|
111
|
+
deleted_at: Date | null;
|
|
112
|
+
};
|
|
113
|
+
export type SearchIndexSeedReason =
|
|
114
|
+
/** Created but never filled. */
|
|
115
|
+
"index_created"
|
|
116
|
+
/** Exists and holds no documents — an engine wiped or restarted underneath us. */
|
|
117
|
+
| "index_empty"
|
|
118
|
+
/** A migration built a new index that has to be filled, then swapped in. */
|
|
119
|
+
| "schema_changed" | "last_run_failed";
|
|
120
|
+
export type SearchIndexSeedAction = {
|
|
121
|
+
index: string;
|
|
122
|
+
/** The version being filled. */
|
|
123
|
+
target_version: SearchIndexVersionRecord;
|
|
124
|
+
/**
|
|
125
|
+
* Whether `target_version` has to become the index's active version once
|
|
126
|
+
* filled, rather than being seeded in place as the already-active version.
|
|
127
|
+
*/
|
|
128
|
+
swap: boolean;
|
|
129
|
+
reason: SearchIndexSeedReason;
|
|
130
|
+
};
|
|
131
|
+
//# sourceMappingURL=index.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/types/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,MAAM,EACN,qBAAqB,EACrB,8BAA8B,EAC9B,eAAe,EACf,WAAW,EACZ,MAAM,wBAAwB,CAAA;AAE/B,eAAO,MAAM,yCAAyC,gCACvB,CAAA;AAG/B,eAAO,MAAM,gCAAgC,SAAS,CAAA;AAEtD,MAAM,MAAM,mBAAmB,GAAG,OAAO,CAAC,8BAA8B,CAAC,GAAG;IAC1E;;;;OAIG;IACH,SAAS,CAAC,EAAE;QACV,OAAO,EAAE,MAAM,GAAG,qBAAqB,CAAA;QACvC,EAAE,EAAE,MAAM,CAAA;QACV,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;KAClC,EAAE,CAAA;IAIH,gBAAgB,CAAC,EAAE,MAAM,CAAA;IAGzB,YAAY,CAAC,EAAE,MAAM,CAAA;IAErB;;;;;;OAMG;IACH,OAAO,CAAC,EAAE,WAAW,CAAC,qBAAqB,EAAE,CAAA;IAE7C,OAAO,CAAC,EAAE;QAGR,UAAU,CAAC,EAAE,MAAM,CAAA;KACpB,CAAA;CACF,CAAA;AAED,OAAO,QAAQ,cAAc,CAAC;IAC5B,UAAU,aAAa;QACrB,eAAe,EAAE,mBAAmB,CAAA;KACrC;CACF;AAED;;;;GAIG;AACH,iDAAiD;AACjD,MAAM,MAAM,aAAa,GAAG,GAAG,CAC7B,MAAM,EACN,WAAW,CAAC,6BAA6B,CAC1C,CAAA;AAED,MAAM,MAAM,kBAAkB,GAAG;IAC/B,SAAS,EAAE,WAAW,CAAC,eAAe,CAAA;IACtC,MAAM,EAAE,MAAM,CAAA;IACd,OAAO,EAAE,mBAAmB,CAAA;IAC5B,OAAO,EAAE,aAAa,CAAA;IACtB,SAAS,EAAE;QAAE,QAAQ,CAAC,UAAU,EAAE,MAAM,GAAG,WAAW,CAAC,eAAe,CAAA;KAAE,CAAA;IACxE,YAAY,EAAE,eAAe,CAAC,mBAAmB,CAAC,GAAG,CAAC,CAAA;IACtD,cAAc,EAAE,eAAe,CAAC,mBAAmB,CAAC,GAAG,CAAC,CAAA;IACxD,WAAW,EAAE,eAAe,CAAC,mBAAmB,CAAC,GAAG,CAAC,CAAA;IAGrD,OAAO,CAAC,EAAE;QACR,OAAO,CAAC,CAAC,EACP,IAAI,EAAE,MAAM,GAAG,MAAM,EAAE,EACvB,GAAG,EAAE,CAAC,MAAM,CAAC,EAAE,WAAW,KAAK,OAAO,CAAC,CAAC,CAAC,EACzC,IAAI,CAAC,EAAE;YAAE,OAAO,CAAC,EAAE,MAAM,CAAC;YAAC,MAAM,CAAC,EAAE,MAAM,CAAA;SAAE,GAC3C,OAAO,CAAC,CAAC,CAAC,CAAA;KACd,CAAA;IAOD,kBAAkB,CAAC,EAAE;QACnB,GAAG,CACD,IAAI,EAAE,MAAM,EACZ,OAAO,CAAC,EAAE;YAAE,KAAK,CAAC,EAAE,OAAO,CAAA;SAAE,GAC5B,OAAO,CAAC;YAAE,aAAa,EAAE,MAAM,CAAC;YAAC,QAAQ,EAAE,MAAM,CAAC;YAAC,OAAO,EAAE,MAAM,CAAA;SAAE,CAAC,CAAA;QACxE,GAAG,CACD,IAAI,EAAE,MAAM,EACZ,KAAK,EAAE;YAAE,aAAa,EAAE,MAAM,CAAC;YAAC,QAAQ,EAAE,MAAM,CAAC;YAAC,OAAO,EAAE,MAAM,CAAA;SAAE,GAClE,OAAO,CAAC,IAAI,CAAC,CAAA;QAChB,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC,CAAA;KAC5B,CAAA;CACF,CAAA;AAGD,MAAM,MAAM,mBAAmB,GAAG,IAAI,CACpC,kBAAkB,EAClB,SAAS,GAAG,cAAc,GAAG,gBAAgB,GAAG,WAAW,GAAG,QAAQ,CACvE,CAAA;AAID,MAAM,MAAM,iBAAiB,GAAG,mBAAmB,GACjD,IAAI,CACF,kBAAkB,EAClB,aAAa,GAAG,WAAW,GAAG,SAAS,GAAG,oBAAoB,CAC/D,CAAA;AAGH,MAAM,MAAM,sBAAsB,GAAG,mBAAmB,GACtD,IAAI,CAAC,kBAAkB,EAAE,WAAW,GAAG,oBAAoB,CAAC,CAAA;AAE9D,kDAAkD;AAClD,MAAM,MAAM,iBAAiB,GAAG,GAAG,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,CAAA;AAIrD,0EAA0E;AAC1E,MAAM,MAAM,iBAAiB,GAAG;IAC9B,EAAE,EAAE,MAAM,CAAA;IACV,IAAI,EAAE,MAAM,CAAA;IACZ,cAAc,EAAE,MAAM,GAAG,IAAI,CAAA;CAC9B,CAAA;AAED,yDAAyD;AACzD,MAAM,MAAM,wBAAwB,GAAG;IACrC,EAAE,EAAE,MAAM,CAAA;IACV,eAAe,EAAE,MAAM,CAAA;IACvB,OAAO,EAAE,MAAM,CAAA;IACf,QAAQ,EAAE,MAAM,CAAA;IAChB,aAAa,EAAE,MAAM,CAAA;IACrB,eAAe,EAAE,MAAM,CAAA;IACvB,MAAM,EAAE,MAAM,CAAA;CACf,CAAA;AAED,qFAAqF;AACrF,MAAM,MAAM,qBAAqB,GAAG;IAClC,EAAE,EAAE,MAAM,CAAA;IACV,uBAAuB,EAAE,MAAM,CAAA;IAC/B,MAAM,EAAE,MAAM,GAAG,IAAI,CAAA;IACrB,MAAM,EAAE,MAAM,CAAA;IACd,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,CAAA;IACvC,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAA;IACvB,wDAAwD;IACxD,SAAS,EAAE,OAAO,CAAA;IAClB,gBAAgB,EAAE,MAAM,CAAA;IACxB,UAAU,EAAE,IAAI,GAAG,IAAI,CAAA;IACvB,YAAY,EAAE,IAAI,GAAG,IAAI,CAAA;IACzB,KAAK,EAAE,MAAM,GAAG,IAAI,CAAA;IACpB,UAAU,EAAE,IAAI,CAAA;IAChB,UAAU,EAAE,IAAI,CAAA;IAChB,UAAU,EAAE,IAAI,GAAG,IAAI,CAAA;CACxB,CAAA;AAQD,MAAM,MAAM,qBAAqB;AAC/B,gCAAgC;AAC9B,eAAe;AACjB,kFAAkF;GAChF,aAAa;AACf,4EAA4E;GAC1E,gBAAgB,GAChB,iBAAiB,CAAA;AAErB,MAAM,MAAM,qBAAqB,GAAG;IAClC,KAAK,EAAE,MAAM,CAAA;IACb,gCAAgC;IAChC,cAAc,EAAE,wBAAwB,CAAA;IACxC;;;OAGG;IACH,IAAI,EAAE,OAAO,CAAA;IACb,MAAM,EAAE,qBAAqB,CAAA;CAC9B,CAAA"}
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.SearchProviderRegistrationPrefix = exports.SearchProviderIdentifiersRegistrationName = void 0;
|
|
4
|
+
exports.SearchProviderIdentifiersRegistrationName = "search_providers_identifier";
|
|
5
|
+
// Providers are registered as `srh_{identifier}_{id}`.
|
|
6
|
+
exports.SearchProviderRegistrationPrefix = "srh_";
|
|
7
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/types/index.ts"],"names":[],"mappings":";;;AAQa,QAAA,yCAAyC,GACpD,6BAA6B,CAAA;AAE/B,uDAAuD;AAC1C,QAAA,gCAAgC,GAAG,MAAM,CAAA"}
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
export type ActiveIndexVersion = {
|
|
2
|
+
physical_name: string;
|
|
3
|
+
provider: string;
|
|
4
|
+
version: number;
|
|
5
|
+
};
|
|
6
|
+
/**
|
|
7
|
+
* Caches which physical index currently serves reads for every logical index,
|
|
8
|
+
* since that can change out from under a running process when another
|
|
9
|
+
* replica finishes seeding and flips it.
|
|
10
|
+
*
|
|
11
|
+
* Populated in one bulk fetch rather than one per index: there are never many
|
|
12
|
+
* search indexes in an app, so listing them all is cheap, and it means
|
|
13
|
+
* refreshing because of one index refreshes every other one for free.
|
|
14
|
+
*
|
|
15
|
+
* Fresh for `SOFT_TTL_MS`. Stale beyond that but still served while a
|
|
16
|
+
* background refresh runs (stale-while-revalidate) — a failed refresh leaves
|
|
17
|
+
* `fetchedAt_` untouched, so the age keeps counting from the last *successful*
|
|
18
|
+
* fetch rather than resetting. Never served past `HARD_TTL_MS` since that last
|
|
19
|
+
* success; a caller waits on a synchronous refetch instead.
|
|
20
|
+
*/
|
|
21
|
+
export declare class ActiveIndexVersionCache {
|
|
22
|
+
protected readonly fetchAll_: () => Promise<Map<string, ActiveIndexVersion>>;
|
|
23
|
+
protected values_: Map<string, ActiveIndexVersion>;
|
|
24
|
+
protected fetchedAt_: number;
|
|
25
|
+
protected refreshing_?: Promise<void>;
|
|
26
|
+
constructor(fetchAll_: () => Promise<Map<string, ActiveIndexVersion>>);
|
|
27
|
+
get(name: string, { fresh }?: {
|
|
28
|
+
fresh?: boolean;
|
|
29
|
+
}): Promise<ActiveIndexVersion>;
|
|
30
|
+
protected refresh(): Promise<void>;
|
|
31
|
+
/** Sets one value directly, e.g. right after this process itself flips it. */
|
|
32
|
+
set(name: string, value: ActiveIndexVersion): Promise<void>;
|
|
33
|
+
invalidate(): Promise<void>;
|
|
34
|
+
protected settle(): Promise<void>;
|
|
35
|
+
}
|
|
36
|
+
//# sourceMappingURL=active-version-cache.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"active-version-cache.d.ts","sourceRoot":"","sources":["../../src/utils/active-version-cache.ts"],"names":[],"mappings":"AAEA,MAAM,MAAM,kBAAkB,GAAG;IAC/B,aAAa,EAAE,MAAM,CAAA;IACrB,QAAQ,EAAE,MAAM,CAAA;IAChB,OAAO,EAAE,MAAM,CAAA;CAChB,CAAA;AAKD;;;;;;;;;;;;;;GAcG;AACH,qBAAa,uBAAuB;IAMhC,SAAS,CAAC,QAAQ,CAAC,SAAS,EAAE,MAAM,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,kBAAkB,CAAC,CAAC;IAL9E,SAAS,CAAC,OAAO,kCAAwC;IACzD,SAAS,CAAC,UAAU,SAAI;IACxB,SAAS,CAAC,WAAW,CAAC,EAAE,OAAO,CAAC,IAAI,CAAC,CAAA;gBAGhB,SAAS,EAAE,MAAM,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,kBAAkB,CAAC,CAAC;IAGxE,GAAG,CACP,IAAI,EAAE,MAAM,EACZ,EAAE,KAAa,EAAE,GAAE;QAAE,KAAK,CAAC,EAAE,OAAO,CAAA;KAAO,GAC1C,OAAO,CAAC,kBAAkB,CAAC;IAwB9B,SAAS,CAAC,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;IAalC,8EAA8E;IACxE,GAAG,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,kBAAkB,GAAG,OAAO,CAAC,IAAI,CAAC;IAK3D,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;cAQjB,MAAM,IAAI,OAAO,CAAC,IAAI,CAAC;CAGxC"}
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.ActiveIndexVersionCache = void 0;
|
|
4
|
+
const utils_1 = require("@etohq/framework/utils");
|
|
5
|
+
const SOFT_TTL_MS = 30_000;
|
|
6
|
+
const HARD_TTL_MS = 2 * 60_000;
|
|
7
|
+
/**
|
|
8
|
+
* Caches which physical index currently serves reads for every logical index,
|
|
9
|
+
* since that can change out from under a running process when another
|
|
10
|
+
* replica finishes seeding and flips it.
|
|
11
|
+
*
|
|
12
|
+
* Populated in one bulk fetch rather than one per index: there are never many
|
|
13
|
+
* search indexes in an app, so listing them all is cheap, and it means
|
|
14
|
+
* refreshing because of one index refreshes every other one for free.
|
|
15
|
+
*
|
|
16
|
+
* Fresh for `SOFT_TTL_MS`. Stale beyond that but still served while a
|
|
17
|
+
* background refresh runs (stale-while-revalidate) — a failed refresh leaves
|
|
18
|
+
* `fetchedAt_` untouched, so the age keeps counting from the last *successful*
|
|
19
|
+
* fetch rather than resetting. Never served past `HARD_TTL_MS` since that last
|
|
20
|
+
* success; a caller waits on a synchronous refetch instead.
|
|
21
|
+
*/
|
|
22
|
+
class ActiveIndexVersionCache {
|
|
23
|
+
constructor(fetchAll_) {
|
|
24
|
+
this.fetchAll_ = fetchAll_;
|
|
25
|
+
this.values_ = new Map();
|
|
26
|
+
this.fetchedAt_ = 0;
|
|
27
|
+
}
|
|
28
|
+
async get(name, { fresh = false } = {}) {
|
|
29
|
+
const age = Date.now() - this.fetchedAt_;
|
|
30
|
+
if (fresh || !this.fetchedAt_ || age >= HARD_TTL_MS) {
|
|
31
|
+
await this.refresh();
|
|
32
|
+
}
|
|
33
|
+
else if (age >= SOFT_TTL_MS) {
|
|
34
|
+
void this.refresh().catch(() => {
|
|
35
|
+
// Left stale on purpose — `fetchedAt_` is untouched, so `HARD_TTL_MS`
|
|
36
|
+
// keeps counting from the last success, not this failure.
|
|
37
|
+
});
|
|
38
|
+
}
|
|
39
|
+
const value = this.values_.get(name);
|
|
40
|
+
if (!value) {
|
|
41
|
+
throw new utils_1.EtoError(utils_1.EtoError.Types.NOT_FOUND, `Search index "${name}" has no active version yet. It has to be migrated and seeded first.`);
|
|
42
|
+
}
|
|
43
|
+
return value;
|
|
44
|
+
}
|
|
45
|
+
refresh() {
|
|
46
|
+
this.refreshing_ ??= this.fetchAll_()
|
|
47
|
+
.then((values) => {
|
|
48
|
+
this.values_ = values;
|
|
49
|
+
this.fetchedAt_ = Date.now();
|
|
50
|
+
})
|
|
51
|
+
.finally(() => {
|
|
52
|
+
this.refreshing_ = undefined;
|
|
53
|
+
});
|
|
54
|
+
return this.refreshing_;
|
|
55
|
+
}
|
|
56
|
+
/** Sets one value directly, e.g. right after this process itself flips it. */
|
|
57
|
+
async set(name, value) {
|
|
58
|
+
await this.settle();
|
|
59
|
+
this.values_.set(name, value);
|
|
60
|
+
}
|
|
61
|
+
async invalidate() {
|
|
62
|
+
await this.settle();
|
|
63
|
+
this.fetchedAt_ = 0;
|
|
64
|
+
this.values_.clear();
|
|
65
|
+
}
|
|
66
|
+
// A fetch in flight may have read the database before the change the caller
|
|
67
|
+
// is about to make, so it has to land first rather than overwrite it later.
|
|
68
|
+
async settle() {
|
|
69
|
+
await this.refreshing_?.catch(() => { });
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
exports.ActiveIndexVersionCache = ActiveIndexVersionCache;
|
|
73
|
+
//# sourceMappingURL=active-version-cache.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"active-version-cache.js","sourceRoot":"","sources":["../../src/utils/active-version-cache.ts"],"names":[],"mappings":";;;AAAA,kDAAiD;AAQjD,MAAM,WAAW,GAAG,MAAM,CAAA;AAC1B,MAAM,WAAW,GAAG,CAAC,GAAG,MAAM,CAAA;AAE9B;;;;;;;;;;;;;;GAcG;AACH,MAAa,uBAAuB;IAKlC,YACqB,SAAyD;QAAzD,cAAS,GAAT,SAAS,CAAgD;QALpE,YAAO,GAAG,IAAI,GAAG,EAA8B,CAAA;QAC/C,eAAU,GAAG,CAAC,CAAA;IAKrB,CAAC;IAEJ,KAAK,CAAC,GAAG,CACP,IAAY,EACZ,EAAE,KAAK,GAAG,KAAK,KAA0B,EAAE;QAE3C,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,UAAU,CAAA;QAExC,IAAI,KAAK,IAAI,CAAC,IAAI,CAAC,UAAU,IAAI,GAAG,IAAI,WAAW,EAAE,CAAC;YACpD,MAAM,IAAI,CAAC,OAAO,EAAE,CAAA;QACtB,CAAC;aAAM,IAAI,GAAG,IAAI,WAAW,EAAE,CAAC;YAC9B,KAAK,IAAI,CAAC,OAAO,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE;gBAC7B,sEAAsE;gBACtE,0DAA0D;YAC5D,CAAC,CAAC,CAAA;QACJ,CAAC;QAED,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;QAEpC,IAAI,CAAC,KAAK,EAAE,CAAC;YACX,MAAM,IAAI,gBAAQ,CAChB,gBAAQ,CAAC,KAAK,CAAC,SAAS,EACxB,iBAAiB,IAAI,sEAAsE,CAC5F,CAAA;QACH,CAAC;QAED,OAAO,KAAK,CAAA;IACd,CAAC;IAES,OAAO;QACf,IAAI,CAAC,WAAW,KAAK,IAAI,CAAC,SAAS,EAAE;aAClC,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE;YACf,IAAI,CAAC,OAAO,GAAG,MAAM,CAAA;YACrB,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,CAAA;QAC9B,CAAC,CAAC;aACD,OAAO,CAAC,GAAG,EAAE;YACZ,IAAI,CAAC,WAAW,GAAG,SAAS,CAAA;QAC9B,CAAC,CAAC,CAAA;QAEJ,OAAO,IAAI,CAAC,WAAW,CAAA;IACzB,CAAC;IAED,8EAA8E;IAC9E,KAAK,CAAC,GAAG,CAAC,IAAY,EAAE,KAAyB;QAC/C,MAAM,IAAI,CAAC,MAAM,EAAE,CAAA;QACnB,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,EAAE,KAAK,CAAC,CAAA;IAC/B,CAAC;IAED,KAAK,CAAC,UAAU;QACd,MAAM,IAAI,CAAC,MAAM,EAAE,CAAA;QACnB,IAAI,CAAC,UAAU,GAAG,CAAC,CAAA;QACnB,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAA;IACtB,CAAC;IAED,4EAA4E;IAC5E,4EAA4E;IAClE,KAAK,CAAC,MAAM;QACpB,MAAM,IAAI,CAAC,WAAW,EAAE,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAA;IACzC,CAAC;CACF;AAlED,0DAkEC"}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import { SearchTypes } from "@etohq/framework/types";
|
|
2
|
+
import { SearchIndexContext, SearchIndexRegistry } from "../types";
|
|
3
|
+
type DeletionContext = SearchIndexRegistry & Pick<SearchIndexContext, "syncService" | "locking" | "activeVersionCache">;
|
|
4
|
+
/**
|
|
5
|
+
* Drops every physical index ever built for a logical index and hard deletes the
|
|
6
|
+
* rows behind them, so the next migration plans a `create` and rebuilds from
|
|
7
|
+
* version 1. Soft deleting would leave the numbering to carry on instead.
|
|
8
|
+
*/
|
|
9
|
+
export declare function deleteIndexEntirely(context: DeletionContext, name: string): Promise<SearchTypes.SearchIndexDeleteResult>;
|
|
10
|
+
export {};
|
|
11
|
+
//# sourceMappingURL=deletion.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"deletion.d.ts","sourceRoot":"","sources":["../../src/utils/deletion.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,wBAAwB,CAAA;AACpD,OAAO,EACL,kBAAkB,EAElB,mBAAmB,EAEpB,MAAM,QAAQ,CAAA;AAGf,KAAK,eAAe,GAAG,mBAAmB,GACxC,IAAI,CAAC,kBAAkB,EAAE,aAAa,GAAG,SAAS,GAAG,oBAAoB,CAAC,CAAA;AAE5E;;;;GAIG;AACH,wBAAsB,mBAAmB,CACvC,OAAO,EAAE,eAAe,EACxB,IAAI,EAAE,MAAM,GACX,OAAO,CAAC,WAAW,CAAC,uBAAuB,CAAC,CAW9C"}
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.deleteIndexEntirely = deleteIndexEntirely;
|
|
4
|
+
const index_1 = require("./index");
|
|
5
|
+
/**
|
|
6
|
+
* Drops every physical index ever built for a logical index and hard deletes the
|
|
7
|
+
* rows behind them, so the next migration plans a `create` and rebuilds from
|
|
8
|
+
* version 1. Soft deleting would leave the numbering to carry on instead.
|
|
9
|
+
*/
|
|
10
|
+
async function deleteIndexEntirely(context, name) {
|
|
11
|
+
(0, index_1.retrieveIndexDefinition)(context.indexes, name);
|
|
12
|
+
// The lock a seed and a reindex take, so this cannot land halfway through one.
|
|
13
|
+
if (context.locking) {
|
|
14
|
+
return await context.locking.execute(`search:seed:${name}`, () => deleteOne(context, name));
|
|
15
|
+
}
|
|
16
|
+
return await deleteOne(context, name);
|
|
17
|
+
}
|
|
18
|
+
async function deleteOne(context, name) {
|
|
19
|
+
const [record] = (await context.indexService.list({ name }, { take: 1, withDeleted: true }));
|
|
20
|
+
if (!record) {
|
|
21
|
+
return { index: name, deleted_versions: 0 };
|
|
22
|
+
}
|
|
23
|
+
// `withDeleted` throughout: a version an earlier cleanup soft deleted still
|
|
24
|
+
// owns a row and a physical index, and leaving either behind is what this is
|
|
25
|
+
// meant to avoid.
|
|
26
|
+
const versions = (await context.versionService.list({ search_index_id: record.id }, { take: null, withDeleted: true }));
|
|
27
|
+
for (const version of versions) {
|
|
28
|
+
try {
|
|
29
|
+
const provider = context.providers.retrieve(version.provider);
|
|
30
|
+
await provider.deleteIndex({ index: version.physical_name });
|
|
31
|
+
}
|
|
32
|
+
catch (error) {
|
|
33
|
+
// The rows go either way: keeping them for an index the engine may no
|
|
34
|
+
// longer have is what stops it from ever being rebuilt.
|
|
35
|
+
context.logger.warn(`[Search] Cannot drop physical search index "${version.physical_name}": ${error.message}`);
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
const versionIds = versions.map((version) => version.id);
|
|
39
|
+
if (versionIds.length) {
|
|
40
|
+
const syncs = (await context.syncService.list({ search_index_version_id: versionIds }, { take: null, select: ["id"], withDeleted: true }));
|
|
41
|
+
if (syncs.length) {
|
|
42
|
+
await context.syncService.delete(syncs.map((sync) => sync.id));
|
|
43
|
+
}
|
|
44
|
+
await context.versionService.delete(versionIds);
|
|
45
|
+
}
|
|
46
|
+
await context.indexService.delete([record.id]);
|
|
47
|
+
await context.activeVersionCache?.invalidate();
|
|
48
|
+
return { index: name, deleted_versions: versions.length };
|
|
49
|
+
}
|
|
50
|
+
//# sourceMappingURL=deletion.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"deletion.js","sourceRoot":"","sources":["../../src/utils/deletion.ts"],"names":[],"mappings":";;AAiBA,kDAcC;AAxBD,mCAAiD;AAKjD;;;;GAIG;AACI,KAAK,UAAU,mBAAmB,CACvC,OAAwB,EACxB,IAAY;IAEZ,IAAA,+BAAuB,EAAC,OAAO,CAAC,OAAO,EAAE,IAAI,CAAC,CAAA;IAE9C,+EAA+E;IAC/E,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC;QACpB,OAAO,MAAM,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC,eAAe,IAAI,EAAE,EAAE,GAAG,EAAE,CAC/D,SAAS,CAAC,OAAO,EAAE,IAAI,CAAC,CACzB,CAAA;IACH,CAAC;IAED,OAAO,MAAM,SAAS,CAAC,OAAO,EAAE,IAAI,CAAC,CAAA;AACvC,CAAC;AAED,KAAK,UAAU,SAAS,CACtB,OAAwB,EACxB,IAAY;IAEZ,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,MAAM,OAAO,CAAC,YAAY,CAAC,IAAI,CAC/C,EAAE,IAAI,EAAE,EACR,EAAE,IAAI,EAAE,CAAC,EAAE,WAAW,EAAE,IAAI,EAAE,CAC/B,CAAwB,CAAA;IAEzB,IAAI,CAAC,MAAM,EAAE,CAAC;QACZ,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,gBAAgB,EAAE,CAAC,EAAE,CAAA;IAC7C,CAAC;IAED,4EAA4E;IAC5E,6EAA6E;IAC7E,kBAAkB;IAClB,MAAM,QAAQ,GAAG,CAAC,MAAM,OAAO,CAAC,cAAc,CAAC,IAAI,CACjD,EAAE,eAAe,EAAE,MAAM,CAAC,EAAE,EAAE,EAC9B,EAAE,IAAI,EAAE,IAAI,EAAE,WAAW,EAAE,IAAI,EAAE,CAClC,CAA+B,CAAA;IAEhC,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAC/B,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,OAAO,CAAC,SAAS,CAAC,QAAQ,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAA;YAE7D,MAAM,QAAQ,CAAC,WAAW,CAAC,EAAE,KAAK,EAAE,OAAO,CAAC,aAAa,EAAE,CAAC,CAAA;QAC9D,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,sEAAsE;YACtE,wDAAwD;YACxD,OAAO,CAAC,MAAM,CAAC,IAAI,CACjB,+CAA+C,OAAO,CAAC,aAAa,MAAM,KAAK,CAAC,OAAO,EAAE,CAC1F,CAAA;QACH,CAAC;IACH,CAAC;IAED,MAAM,UAAU,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,EAAE,CAAC,CAAA;IAExD,IAAI,UAAU,CAAC,MAAM,EAAE,CAAC;QACtB,MAAM,KAAK,GAAG,CAAC,MAAM,OAAO,CAAC,WAAW,CAAC,IAAI,CAC3C,EAAE,uBAAuB,EAAE,UAAU,EAAE,EACvC,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC,IAAI,CAAC,EAAE,WAAW,EAAE,IAAI,EAAE,CAClD,CAAqB,CAAA;QAEtB,IAAI,KAAK,CAAC,MAAM,EAAE,CAAC;YACjB,MAAM,OAAO,CAAC,WAAW,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAA;QAChE,CAAC;QAED,MAAM,OAAO,CAAC,cAAc,CAAC,MAAM,CAAC,UAAU,CAAC,CAAA;IACjD,CAAC;IAED,MAAM,OAAO,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAA;IAE9C,MAAM,OAAO,CAAC,kBAAkB,EAAE,UAAU,EAAE,CAAA;IAE9C,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,gBAAgB,EAAE,QAAQ,CAAC,MAAM,EAAE,CAAA;AAC3D,CAAC"}
|
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
import { SearchTypes } from "@etohq/framework/types";
|
|
2
|
+
import { SearchIndexContext, SearchIndexes, SearchIndexVersionRecord } from "../types";
|
|
3
|
+
export declare enum SearchIndexState {
|
|
4
|
+
PENDING = "pending",
|
|
5
|
+
BUILDING = "building",
|
|
6
|
+
READY = "ready",
|
|
7
|
+
ERROR = "error"
|
|
8
|
+
}
|
|
9
|
+
export declare enum SearchSyncStatus {
|
|
10
|
+
PENDING = "pending",
|
|
11
|
+
PROCESSING = "processing",
|
|
12
|
+
DONE = "done",
|
|
13
|
+
FAILED = "failed",
|
|
14
|
+
CANCELED = "canceled"
|
|
15
|
+
}
|
|
16
|
+
export declare const DEFAULT_REINDEX_BATCH_SIZE = 100;
|
|
17
|
+
export type FlatSearchField = {
|
|
18
|
+
path: string;
|
|
19
|
+
field: SearchTypes.SearchFieldDefinition;
|
|
20
|
+
};
|
|
21
|
+
export declare function flattenFields(fields: Record<string, SearchTypes.SearchFieldDefinition>, prefix?: string): FlatSearchField[];
|
|
22
|
+
export declare function isSearchable(field: SearchTypes.SearchFieldDefinition): boolean;
|
|
23
|
+
export declare function isFacetable(field: SearchTypes.SearchFieldDefinition, kind: SearchTypes.SearchFacetKind): boolean;
|
|
24
|
+
/**
|
|
25
|
+
* The dotted paths an index can hand back on hits. Object containers are
|
|
26
|
+
* excluded — only their declared leaves are stored, so returning the container
|
|
27
|
+
* would silently hand back a partial object.
|
|
28
|
+
*/
|
|
29
|
+
export declare function listRetrievablePaths(fields: Record<string, SearchTypes.SearchFieldDefinition>): string[];
|
|
30
|
+
/**
|
|
31
|
+
* Leaf fields the index stores, with the capabilities declared on each. Object
|
|
32
|
+
* containers are omitted — only their leaves are indexed.
|
|
33
|
+
*/
|
|
34
|
+
export declare function listIndexedFields(fields: Record<string, SearchTypes.SearchFieldDefinition>): SearchTypes.SearchIndexFieldInfo[];
|
|
35
|
+
export declare function normalizeSearchQuery({ query, index, }: {
|
|
36
|
+
query: SearchTypes.SearchQuery;
|
|
37
|
+
index: SearchTypes.ResolvedSearchIndexDefinition;
|
|
38
|
+
}): SearchTypes.ProviderSearchQuery;
|
|
39
|
+
/**
|
|
40
|
+
* The base query plus one per faceted field with that field's own filter dropped.
|
|
41
|
+
* Every engine reviewed needs a query per facet, so the fan-out lives here rather
|
|
42
|
+
* than in each provider. The expanded set is then handed to `provider.searchMany`
|
|
43
|
+
* so the engine can pack them into one round-trip. `disjunctive_facets` is
|
|
44
|
+
* stripped from what is handed on — a provider should never see an option it is
|
|
45
|
+
* not expected to act on.
|
|
46
|
+
*/
|
|
47
|
+
export declare function buildDisjunctiveFacetQueries(query: SearchTypes.ProviderSearchQuery): {
|
|
48
|
+
base: SearchTypes.ProviderSearchQuery;
|
|
49
|
+
per_facet: Map<string, SearchTypes.ProviderSearchQuery>;
|
|
50
|
+
};
|
|
51
|
+
export declare function mergeDisjunctiveFacetResults({ base, per_facet, }: {
|
|
52
|
+
base: SearchTypes.SearchResult;
|
|
53
|
+
per_facet: Map<string, SearchTypes.SearchResult>;
|
|
54
|
+
}): SearchTypes.SearchResult;
|
|
55
|
+
export declare function validateFieldUsage({ index, query, }: {
|
|
56
|
+
index: SearchTypes.ResolvedSearchIndexDefinition;
|
|
57
|
+
query: Pick<SearchTypes.SearchQuery, "filters" | "pagination" | "search_options">;
|
|
58
|
+
}): void;
|
|
59
|
+
/**
|
|
60
|
+
* Resolves every definition once, at construction. Duplicates and incoherent
|
|
61
|
+
* definitions fail here, so everything downstream can treat the result as given.
|
|
62
|
+
*/
|
|
63
|
+
export declare function resolveIndexDefinitions({ definitions, default_provider, index_prefix, }: {
|
|
64
|
+
definitions: SearchTypes.SearchIndexDefinition[];
|
|
65
|
+
default_provider: string;
|
|
66
|
+
index_prefix?: string;
|
|
67
|
+
}): SearchIndexes;
|
|
68
|
+
/** Every version of every listed index, grouped by the index's id, newest first. */
|
|
69
|
+
export declare function listVersionsByIndexId(context: Pick<SearchIndexContext, "versionService">, indexIds: string[]): Promise<Map<string, SearchIndexVersionRecord[]>>;
|
|
70
|
+
/**
|
|
71
|
+
* The definition merged with whichever physical index and provider currently
|
|
72
|
+
* serve reads/writes. Every *live* operation (a direct write, an event
|
|
73
|
+
* ingested, a query) must resolve through this rather than
|
|
74
|
+
* `definition.physical_name` directly — that field is only the root a
|
|
75
|
+
* version's name is derived from, and which version is active can change out
|
|
76
|
+
* from under this process.
|
|
77
|
+
*/
|
|
78
|
+
export declare function resolveActiveDefinition(context: Pick<SearchIndexContext, "indexes" | "activeVersionCache">, name: string, { fresh }?: {
|
|
79
|
+
fresh?: boolean;
|
|
80
|
+
}): Promise<SearchTypes.ResolvedSearchIndexDefinition>;
|
|
81
|
+
/**
|
|
82
|
+
* Runs an operation against whichever version is active, and gives it a second
|
|
83
|
+
* go on a `NOT_FOUND` with the cache invalidated first.
|
|
84
|
+
*/
|
|
85
|
+
export declare function withActiveIndexRetry<T>(context: Pick<SearchIndexContext, "activeVersionCache">, run: () => Promise<T>): Promise<T>;
|
|
86
|
+
export declare function retrieveIndexDefinition(indexes: SearchIndexes, name: string): SearchTypes.ResolvedSearchIndexDefinition;
|
|
87
|
+
export declare function assertTaskAccepted(task: SearchTypes.SearchTask, index: string): SearchTypes.SearchTask;
|
|
88
|
+
//# sourceMappingURL=index.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/utils/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,wBAAwB,CAAA;AAEpD,OAAO,EACL,kBAAkB,EAClB,aAAa,EACb,wBAAwB,EACzB,MAAM,QAAQ,CAAA;AAGf,oBAAY,gBAAgB;IAC1B,OAAO,YAAY;IACnB,QAAQ,aAAa;IACrB,KAAK,UAAU;IACf,KAAK,UAAU;CAChB;AAED,oBAAY,gBAAgB;IAC1B,OAAO,YAAY;IACnB,UAAU,eAAe;IACzB,IAAI,SAAS;IACb,MAAM,WAAW;IACjB,QAAQ,aAAa;CACtB;AAGD,eAAO,MAAM,0BAA0B,MAAM,CAAA;AAI7C,MAAM,MAAM,eAAe,GAAG;IAC5B,IAAI,EAAE,MAAM,CAAA;IACZ,KAAK,EAAE,WAAW,CAAC,qBAAqB,CAAA;CACzC,CAAA;AAID,wBAAgB,aAAa,CAC3B,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,WAAW,CAAC,qBAAqB,CAAC,EACzD,MAAM,SAAK,GACV,eAAe,EAAE,CAanB;AAQD,wBAAgB,YAAY,CAC1B,KAAK,EAAE,WAAW,CAAC,qBAAqB,GACvC,OAAO,CAET;AAED,wBAAgB,WAAW,CACzB,KAAK,EAAE,WAAW,CAAC,qBAAqB,EACxC,IAAI,EAAE,WAAW,CAAC,eAAe,GAChC,OAAO,CAQT;AA+BD;;;;GAIG;AACH,wBAAgB,oBAAoB,CAClC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,WAAW,CAAC,qBAAqB,CAAC,GACxD,MAAM,EAAE,CAIV;AAED;;;GAGG;AACH,wBAAgB,iBAAiB,CAC/B,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,WAAW,CAAC,qBAAqB,CAAC,GACxD,WAAW,CAAC,oBAAoB,EAAE,CAYpC;AA2DD,wBAAgB,oBAAoB,CAAC,EACnC,KAAK,EACL,KAAK,GACN,EAAE;IACD,KAAK,EAAE,WAAW,CAAC,WAAW,CAAA;IAC9B,KAAK,EAAE,WAAW,CAAC,6BAA6B,CAAA;CACjD,GAAG,WAAW,CAAC,mBAAmB,CA6ClC;AA+CD;;;;;;;GAOG;AACH,wBAAgB,4BAA4B,CAC1C,KAAK,EAAE,WAAW,CAAC,mBAAmB,GACrC;IACD,IAAI,EAAE,WAAW,CAAC,mBAAmB,CAAA;IACrC,SAAS,EAAE,GAAG,CAAC,MAAM,EAAE,WAAW,CAAC,mBAAmB,CAAC,CAAA;CACxD,CA6BA;AAED,wBAAgB,4BAA4B,CAAC,EAC3C,IAAI,EACJ,SAAS,GACV,EAAE;IACD,IAAI,EAAE,WAAW,CAAC,YAAY,CAAA;IAC9B,SAAS,EAAE,GAAG,CAAC,MAAM,EAAE,WAAW,CAAC,YAAY,CAAC,CAAA;CACjD,GAAG,WAAW,CAAC,YAAY,CAe3B;AAsBD,wBAAgB,kBAAkB,CAAC,EACjC,KAAK,EACL,KAAK,GACN,EAAE;IACD,KAAK,EAAE,WAAW,CAAC,6BAA6B,CAAA;IAGhD,KAAK,EAAE,IAAI,CACT,WAAW,CAAC,WAAW,EACvB,SAAS,GAAG,YAAY,GAAG,gBAAgB,CAC5C,CAAA;CACF,GAAG,IAAI,CAsHP;AAED;;;GAGG;AACH,wBAAgB,uBAAuB,CAAC,EACtC,WAAW,EACX,gBAAgB,EAChB,YAAY,GACb,EAAE;IACD,WAAW,EAAE,WAAW,CAAC,qBAAqB,EAAE,CAAA;IAChD,gBAAgB,EAAE,MAAM,CAAA;IACxB,YAAY,CAAC,EAAE,MAAM,CAAA;CACtB,GAAG,aAAa,CAuDhB;AA4ED,oFAAoF;AACpF,wBAAsB,qBAAqB,CACzC,OAAO,EAAE,IAAI,CAAC,kBAAkB,EAAE,gBAAgB,CAAC,EACnD,QAAQ,EAAE,MAAM,EAAE,GACjB,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,wBAAwB,EAAE,CAAC,CAAC,CAmBlD;AAED;;;;;;;GAOG;AACH,wBAAsB,uBAAuB,CAC3C,OAAO,EAAE,IAAI,CAAC,kBAAkB,EAAE,SAAS,GAAG,oBAAoB,CAAC,EACnE,IAAI,EAAE,MAAM,EACZ,EAAE,KAAa,EAAE,GAAE;IAAE,KAAK,CAAC,EAAE,OAAO,CAAA;CAAO,GAC1C,OAAO,CAAC,WAAW,CAAC,6BAA6B,CAAC,CAcpD;AAED;;;GAGG;AACH,wBAAsB,oBAAoB,CAAC,CAAC,EAC1C,OAAO,EAAE,IAAI,CAAC,kBAAkB,EAAE,oBAAoB,CAAC,EACvD,GAAG,EAAE,MAAM,OAAO,CAAC,CAAC,CAAC,GACpB,OAAO,CAAC,CAAC,CAAC,CAgBZ;AAED,wBAAgB,uBAAuB,CACrC,OAAO,EAAE,aAAa,EACtB,IAAI,EAAE,MAAM,GACX,WAAW,CAAC,6BAA6B,CAa3C;AAED,wBAAgB,kBAAkB,CAChC,IAAI,EAAE,WAAW,CAAC,UAAU,EAC5B,KAAK,EAAE,MAAM,GACZ,WAAW,CAAC,UAAU,CAWxB"}
|