@tinacms/search 0.0.0-e623534-20250401065516 → 0.0.0-e96d36b-20251210035710
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index-client.d.ts +3 -1
- package/dist/index-client.js +183 -195
- package/dist/index.js +59 -78
- package/package.json +11 -12
- package/dist/index-client.mjs +0 -195
package/dist/index-client.d.ts
CHANGED
|
@@ -4,7 +4,9 @@ export declare const queryToSearchIndexQuery: (query: string, stopwordLanguages?
|
|
|
4
4
|
export declare const optionsToSearchIndexOptions: (options?: {
|
|
5
5
|
limit?: number;
|
|
6
6
|
cursor?: string;
|
|
7
|
-
}) => {
|
|
7
|
+
}) => {
|
|
8
|
+
PAGE?: {};
|
|
9
|
+
};
|
|
8
10
|
export declare const parseSearchIndexResponse: (data: any, options?: {
|
|
9
11
|
limit?: number;
|
|
10
12
|
cursor?: string;
|
package/dist/index-client.js
CHANGED
|
@@ -1,215 +1,203 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
"
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
}
|
|
16
|
-
}
|
|
17
|
-
}
|
|
18
|
-
n.default = e;
|
|
19
|
-
return Object.freeze(n);
|
|
1
|
+
var __defProp = Object.defineProperty;
|
|
2
|
+
var __defNormalProp = (obj, key, value) => key in obj ? __defProp(obj, key, { enumerable: true, configurable: true, writable: true, value }) : obj[key] = value;
|
|
3
|
+
var __publicField = (obj, key, value) => {
|
|
4
|
+
__defNormalProp(obj, typeof key !== "symbol" ? key + "" : key, value);
|
|
5
|
+
return value;
|
|
6
|
+
};
|
|
7
|
+
import * as sw from "stopword";
|
|
8
|
+
class StringBuilder {
|
|
9
|
+
constructor(limit) {
|
|
10
|
+
__publicField(this, "buffer");
|
|
11
|
+
__publicField(this, "length", 0);
|
|
12
|
+
__publicField(this, "limit");
|
|
13
|
+
this.buffer = [];
|
|
14
|
+
this.limit = limit;
|
|
20
15
|
}
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
this.buffer
|
|
26
|
-
this.
|
|
27
|
-
|
|
28
|
-
append(str) {
|
|
29
|
-
if (this.length + str.length > this.limit) {
|
|
16
|
+
append(str) {
|
|
17
|
+
if (this.length + str.length > this.limit) {
|
|
18
|
+
return true;
|
|
19
|
+
} else {
|
|
20
|
+
this.buffer.push(str);
|
|
21
|
+
this.length += str.length;
|
|
22
|
+
if (this.length > this.limit) {
|
|
30
23
|
return true;
|
|
31
|
-
} else {
|
|
32
|
-
this.buffer.push(str);
|
|
33
|
-
this.length += str.length;
|
|
34
|
-
if (this.length > this.limit) {
|
|
35
|
-
return true;
|
|
36
|
-
}
|
|
37
|
-
return false;
|
|
38
24
|
}
|
|
25
|
+
return false;
|
|
39
26
|
}
|
|
40
|
-
toString() {
|
|
41
|
-
return this.buffer.join(" ");
|
|
42
|
-
}
|
|
43
27
|
}
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
28
|
+
toString() {
|
|
29
|
+
return this.buffer.join(" ");
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
const extractText = (data, acc, indexableNodeTypes) => {
|
|
33
|
+
var _a, _b;
|
|
34
|
+
if (data) {
|
|
35
|
+
if (indexableNodeTypes.indexOf(data.type) !== -1 && (data.text || data.value)) {
|
|
36
|
+
const tokens = tokenizeString(data.text || data.value);
|
|
37
|
+
for (const token of tokens) {
|
|
38
|
+
if (acc.append(token)) {
|
|
39
|
+
return;
|
|
53
40
|
}
|
|
54
41
|
}
|
|
55
|
-
(_b = (_a = data.children) == null ? void 0 : _a.forEach) == null ? void 0 : _b.call(
|
|
56
|
-
_a,
|
|
57
|
-
(child) => extractText(child, acc, indexableNodeTypes)
|
|
58
|
-
);
|
|
59
42
|
}
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
43
|
+
(_b = (_a = data.children) == null ? void 0 : _a.forEach) == null ? void 0 : _b.call(
|
|
44
|
+
_a,
|
|
45
|
+
(child) => extractText(child, acc, indexableNodeTypes)
|
|
46
|
+
);
|
|
47
|
+
}
|
|
48
|
+
};
|
|
49
|
+
const relativePath = (path, collection) => {
|
|
50
|
+
return path.replace(/\\/g, "/").replace(collection.path, "").replace(/^\/|\/$/g, "");
|
|
51
|
+
};
|
|
52
|
+
const tokenizeString = (str) => {
|
|
53
|
+
return str.split(/[\s\.,]+/).map((s) => s.toLowerCase()).filter((s) => s);
|
|
54
|
+
};
|
|
55
|
+
const processTextFieldValue = (value, maxLen) => {
|
|
56
|
+
const tokens = tokenizeString(value);
|
|
57
|
+
const builder = new StringBuilder(maxLen);
|
|
58
|
+
for (const part of tokens) {
|
|
59
|
+
if (builder.append(part)) {
|
|
60
|
+
break;
|
|
74
61
|
}
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
62
|
+
}
|
|
63
|
+
return builder.toString();
|
|
64
|
+
};
|
|
65
|
+
const processDocumentForIndexing = (data, path, collection, textIndexLength, field) => {
|
|
66
|
+
if (!field) {
|
|
67
|
+
const relPath = relativePath(path, collection);
|
|
68
|
+
data["_id"] = `${collection.name}:${relPath}`;
|
|
69
|
+
data["_relativePath"] = relPath;
|
|
70
|
+
}
|
|
71
|
+
for (const f of (field == null ? void 0 : field.fields) || collection.fields || []) {
|
|
72
|
+
if (!f.searchable) {
|
|
73
|
+
delete data[f.name];
|
|
74
|
+
continue;
|
|
82
75
|
}
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
if (f.type === "object") {
|
|
91
|
-
if (isList) {
|
|
92
|
-
data[f.name] = data[f.name].map(
|
|
93
|
-
(obj) => processDocumentForIndexing(
|
|
94
|
-
obj,
|
|
95
|
-
path,
|
|
96
|
-
collection,
|
|
97
|
-
textIndexLength,
|
|
98
|
-
f
|
|
99
|
-
)
|
|
100
|
-
);
|
|
101
|
-
} else {
|
|
102
|
-
data[f.name] = processDocumentForIndexing(
|
|
103
|
-
data[f.name],
|
|
76
|
+
const isList = f.list;
|
|
77
|
+
if (data[f.name]) {
|
|
78
|
+
if (f.type === "object") {
|
|
79
|
+
if (isList) {
|
|
80
|
+
data[f.name] = data[f.name].map(
|
|
81
|
+
(obj) => processDocumentForIndexing(
|
|
82
|
+
obj,
|
|
104
83
|
path,
|
|
105
84
|
collection,
|
|
106
85
|
textIndexLength,
|
|
107
86
|
f
|
|
108
|
-
)
|
|
109
|
-
|
|
110
|
-
} else
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
87
|
+
)
|
|
88
|
+
);
|
|
89
|
+
} else {
|
|
90
|
+
data[f.name] = processDocumentForIndexing(
|
|
91
|
+
data[f.name],
|
|
92
|
+
path,
|
|
93
|
+
collection,
|
|
94
|
+
textIndexLength,
|
|
95
|
+
f
|
|
96
|
+
);
|
|
97
|
+
}
|
|
98
|
+
} else if (f.type === "string") {
|
|
99
|
+
const fieldTextIndexLength = f.maxSearchIndexFieldLength || textIndexLength;
|
|
100
|
+
if (isList) {
|
|
101
|
+
data[f.name] = data[f.name].map(
|
|
102
|
+
(value) => processTextFieldValue(value, fieldTextIndexLength)
|
|
103
|
+
);
|
|
104
|
+
} else {
|
|
105
|
+
data[f.name] = processTextFieldValue(
|
|
106
|
+
data[f.name],
|
|
107
|
+
fieldTextIndexLength
|
|
108
|
+
);
|
|
109
|
+
}
|
|
110
|
+
} else if (f.type === "rich-text") {
|
|
111
|
+
const fieldTextIndexLength = f.maxSearchIndexFieldLength || textIndexLength;
|
|
112
|
+
if (isList) {
|
|
113
|
+
data[f.name] = data[f.name].map((value) => {
|
|
131
114
|
const acc = new StringBuilder(fieldTextIndexLength);
|
|
132
|
-
extractText(
|
|
133
|
-
|
|
134
|
-
}
|
|
115
|
+
extractText(value, acc, ["text", "code_block", "html"]);
|
|
116
|
+
return acc.toString();
|
|
117
|
+
});
|
|
118
|
+
} else {
|
|
119
|
+
const acc = new StringBuilder(fieldTextIndexLength);
|
|
120
|
+
extractText(data[f.name], acc, ["text", "code_block", "html"]);
|
|
121
|
+
data[f.name] = acc.toString();
|
|
135
122
|
}
|
|
136
123
|
}
|
|
137
124
|
}
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
stopwords = [];
|
|
148
|
-
for (const key of keys) {
|
|
149
|
-
stopwords.push(...sw__namespace[key]);
|
|
150
|
-
}
|
|
151
|
-
memo[keys.join(",")] = stopwords;
|
|
152
|
-
}
|
|
153
|
-
return stopwords;
|
|
154
|
-
};
|
|
155
|
-
const queryToSearchIndexQuery = (query, stopwordLanguages) => {
|
|
156
|
-
let q;
|
|
157
|
-
const parts = query.split(" ");
|
|
158
|
-
const stopwords = lookupStopwords(stopwordLanguages);
|
|
159
|
-
if (parts.length === 1) {
|
|
160
|
-
q = { AND: [parts[0]] };
|
|
161
|
-
} else {
|
|
162
|
-
q = {
|
|
163
|
-
AND: parts.filter(
|
|
164
|
-
(part) => part.toLowerCase() !== "and" && stopwords.indexOf(part.toLowerCase()) === -1
|
|
165
|
-
)
|
|
166
|
-
};
|
|
167
|
-
}
|
|
168
|
-
return q;
|
|
169
|
-
};
|
|
170
|
-
const optionsToSearchIndexOptions = (options) => {
|
|
171
|
-
const opt = {};
|
|
172
|
-
if (options == null ? void 0 : options.limit) {
|
|
173
|
-
opt["PAGE"] = {
|
|
174
|
-
SIZE: options.limit,
|
|
175
|
-
NUMBER: (options == null ? void 0 : options.cursor) ? parseInt(options.cursor) : 0
|
|
176
|
-
};
|
|
125
|
+
}
|
|
126
|
+
return data;
|
|
127
|
+
};
|
|
128
|
+
const memo = {};
|
|
129
|
+
const lookupStopwords = (keys, defaultStopWords = sw.eng) => {
|
|
130
|
+
let stopwords = defaultStopWords;
|
|
131
|
+
if (keys) {
|
|
132
|
+
if (memo[keys.join(",")]) {
|
|
133
|
+
return memo[keys.join(",")];
|
|
177
134
|
}
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
const results = data["RESULT"];
|
|
182
|
-
const total = data["RESULT_LENGTH"];
|
|
183
|
-
if ((options == null ? void 0 : options.cursor) && (options == null ? void 0 : options.limit)) {
|
|
184
|
-
const prevCursor = options.cursor === "0" ? null : (parseInt(options.cursor) - 1).toString();
|
|
185
|
-
const nextCursor = total <= (parseInt(options.cursor) + 1) * options.limit ? null : (parseInt(options.cursor) + 1).toString();
|
|
186
|
-
return {
|
|
187
|
-
results,
|
|
188
|
-
total,
|
|
189
|
-
prevCursor,
|
|
190
|
-
nextCursor
|
|
191
|
-
};
|
|
192
|
-
} else if (!(options == null ? void 0 : options.cursor) && (options == null ? void 0 : options.limit)) {
|
|
193
|
-
const prevCursor = null;
|
|
194
|
-
const nextCursor = total <= options.limit ? null : "1";
|
|
195
|
-
return {
|
|
196
|
-
results,
|
|
197
|
-
total,
|
|
198
|
-
prevCursor,
|
|
199
|
-
nextCursor
|
|
200
|
-
};
|
|
201
|
-
} else {
|
|
202
|
-
return {
|
|
203
|
-
results,
|
|
204
|
-
total,
|
|
205
|
-
prevCursor: null,
|
|
206
|
-
nextCursor: null
|
|
207
|
-
};
|
|
135
|
+
stopwords = [];
|
|
136
|
+
for (const key of keys) {
|
|
137
|
+
stopwords.push(...sw[key]);
|
|
208
138
|
}
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
139
|
+
memo[keys.join(",")] = stopwords;
|
|
140
|
+
}
|
|
141
|
+
return stopwords;
|
|
142
|
+
};
|
|
143
|
+
const queryToSearchIndexQuery = (query, stopwordLanguages) => {
|
|
144
|
+
let q;
|
|
145
|
+
const parts = query.split(" ");
|
|
146
|
+
const stopwords = lookupStopwords(stopwordLanguages);
|
|
147
|
+
if (parts.length === 1) {
|
|
148
|
+
q = { AND: [parts[0]] };
|
|
149
|
+
} else {
|
|
150
|
+
q = {
|
|
151
|
+
AND: parts.filter(
|
|
152
|
+
(part) => part.toLowerCase() !== "and" && stopwords.indexOf(part.toLowerCase()) === -1
|
|
153
|
+
)
|
|
154
|
+
};
|
|
155
|
+
}
|
|
156
|
+
return q;
|
|
157
|
+
};
|
|
158
|
+
const optionsToSearchIndexOptions = (options) => {
|
|
159
|
+
const opt = {};
|
|
160
|
+
if (options == null ? void 0 : options.limit) {
|
|
161
|
+
opt["PAGE"] = {
|
|
162
|
+
SIZE: options.limit,
|
|
163
|
+
NUMBER: (options == null ? void 0 : options.cursor) ? parseInt(options.cursor) : 0
|
|
164
|
+
};
|
|
165
|
+
}
|
|
166
|
+
return opt;
|
|
167
|
+
};
|
|
168
|
+
const parseSearchIndexResponse = (data, options) => {
|
|
169
|
+
const results = data["RESULT"];
|
|
170
|
+
const total = data["RESULT_LENGTH"];
|
|
171
|
+
if ((options == null ? void 0 : options.cursor) && (options == null ? void 0 : options.limit)) {
|
|
172
|
+
const prevCursor = options.cursor === "0" ? null : (parseInt(options.cursor) - 1).toString();
|
|
173
|
+
const nextCursor = total <= (parseInt(options.cursor) + 1) * options.limit ? null : (parseInt(options.cursor) + 1).toString();
|
|
174
|
+
return {
|
|
175
|
+
results,
|
|
176
|
+
total,
|
|
177
|
+
prevCursor,
|
|
178
|
+
nextCursor
|
|
179
|
+
};
|
|
180
|
+
} else if (!(options == null ? void 0 : options.cursor) && (options == null ? void 0 : options.limit)) {
|
|
181
|
+
const prevCursor = null;
|
|
182
|
+
const nextCursor = total <= options.limit ? null : "1";
|
|
183
|
+
return {
|
|
184
|
+
results,
|
|
185
|
+
total,
|
|
186
|
+
prevCursor,
|
|
187
|
+
nextCursor
|
|
188
|
+
};
|
|
189
|
+
} else {
|
|
190
|
+
return {
|
|
191
|
+
results,
|
|
192
|
+
total,
|
|
193
|
+
prevCursor: null,
|
|
194
|
+
nextCursor: null
|
|
195
|
+
};
|
|
196
|
+
}
|
|
197
|
+
};
|
|
198
|
+
export {
|
|
199
|
+
optionsToSearchIndexOptions,
|
|
200
|
+
parseSearchIndexResponse,
|
|
201
|
+
processDocumentForIndexing,
|
|
202
|
+
queryToSearchIndexQuery
|
|
203
|
+
};
|
package/dist/index.js
CHANGED
|
@@ -1,50 +1,23 @@
|
|
|
1
|
-
var __create = Object.create;
|
|
2
|
-
var __defProp = Object.defineProperty;
|
|
3
|
-
var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
|
|
4
|
-
var __getOwnPropNames = Object.getOwnPropertyNames;
|
|
5
|
-
var __getProtoOf = Object.getPrototypeOf;
|
|
6
|
-
var __hasOwnProp = Object.prototype.hasOwnProperty;
|
|
7
|
-
var __export = (target, all) => {
|
|
8
|
-
for (var name in all)
|
|
9
|
-
__defProp(target, name, { get: all[name], enumerable: true });
|
|
10
|
-
};
|
|
11
|
-
var __copyProps = (to, from, except, desc) => {
|
|
12
|
-
if (from && typeof from === "object" || typeof from === "function") {
|
|
13
|
-
for (let key of __getOwnPropNames(from))
|
|
14
|
-
if (!__hasOwnProp.call(to, key) && key !== except)
|
|
15
|
-
__defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
|
|
16
|
-
}
|
|
17
|
-
return to;
|
|
18
|
-
};
|
|
19
|
-
var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
|
|
20
|
-
// If the importer is in node compatibility mode or this is not an ESM
|
|
21
|
-
// file that has been converted to a CommonJS file using a Babel-
|
|
22
|
-
// compatible transform (i.e. "__esModule" has not been set), then set
|
|
23
|
-
// "default" to the CommonJS "module.exports" for node compatibility.
|
|
24
|
-
isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
|
|
25
|
-
mod
|
|
26
|
-
));
|
|
27
|
-
var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
|
|
28
|
-
|
|
29
1
|
// src/index.ts
|
|
30
|
-
|
|
31
|
-
__export(index_exports, {
|
|
32
|
-
LocalSearchIndexClient: () => LocalSearchIndexClient,
|
|
33
|
-
SearchIndexer: () => SearchIndexer,
|
|
34
|
-
TinaCMSSearchIndexClient: () => TinaCMSSearchIndexClient,
|
|
35
|
-
si: () => import_search_index2.default
|
|
36
|
-
});
|
|
37
|
-
module.exports = __toCommonJS(index_exports);
|
|
38
|
-
var import_search_index2 = __toESM(require("search-index"));
|
|
2
|
+
import si2 from "search-index";
|
|
39
3
|
|
|
40
4
|
// src/indexer/index.ts
|
|
41
|
-
|
|
5
|
+
import {
|
|
6
|
+
loadAndParseWithAliases,
|
|
7
|
+
sequential,
|
|
8
|
+
scanAllContent,
|
|
9
|
+
scanContentByPaths,
|
|
10
|
+
transformDocument,
|
|
11
|
+
transformDocumentIntoPayload
|
|
12
|
+
} from "@tinacms/graphql";
|
|
42
13
|
|
|
43
14
|
// src/indexer/utils.ts
|
|
44
|
-
|
|
15
|
+
import * as sw from "stopword";
|
|
45
16
|
var StringBuilder = class {
|
|
17
|
+
buffer;
|
|
18
|
+
length = 0;
|
|
19
|
+
limit;
|
|
46
20
|
constructor(limit) {
|
|
47
|
-
this.length = 0;
|
|
48
21
|
this.buffer = [];
|
|
49
22
|
this.limit = limit;
|
|
50
23
|
}
|
|
@@ -65,7 +38,6 @@ var StringBuilder = class {
|
|
|
65
38
|
}
|
|
66
39
|
};
|
|
67
40
|
var extractText = (data, acc, indexableNodeTypes) => {
|
|
68
|
-
var _a, _b;
|
|
69
41
|
if (data) {
|
|
70
42
|
if (indexableNodeTypes.indexOf(data.type) !== -1 && (data.text || data.value)) {
|
|
71
43
|
const tokens = tokenizeString(data.text || data.value);
|
|
@@ -75,8 +47,7 @@ var extractText = (data, acc, indexableNodeTypes) => {
|
|
|
75
47
|
}
|
|
76
48
|
}
|
|
77
49
|
}
|
|
78
|
-
|
|
79
|
-
_a,
|
|
50
|
+
data.children?.forEach?.(
|
|
80
51
|
(child) => extractText(child, acc, indexableNodeTypes)
|
|
81
52
|
);
|
|
82
53
|
}
|
|
@@ -103,7 +74,7 @@ var processDocumentForIndexing = (data, path, collection, textIndexLength, field
|
|
|
103
74
|
data["_id"] = `${collection.name}:${relPath}`;
|
|
104
75
|
data["_relativePath"] = relPath;
|
|
105
76
|
}
|
|
106
|
-
for (const f of
|
|
77
|
+
for (const f of field?.fields || collection.fields || []) {
|
|
107
78
|
if (!f.searchable) {
|
|
108
79
|
delete data[f.name];
|
|
109
80
|
continue;
|
|
@@ -178,6 +149,11 @@ var lookupStopwords = (keys, defaultStopWords = sw.eng) => {
|
|
|
178
149
|
|
|
179
150
|
// src/indexer/index.ts
|
|
180
151
|
var SearchIndexer = class {
|
|
152
|
+
batchSize;
|
|
153
|
+
client;
|
|
154
|
+
bridge;
|
|
155
|
+
schema;
|
|
156
|
+
textIndexLength;
|
|
181
157
|
constructor(options) {
|
|
182
158
|
this.client = options.client;
|
|
183
159
|
this.bridge = options.bridge;
|
|
@@ -188,12 +164,12 @@ var SearchIndexer = class {
|
|
|
188
164
|
makeIndexerCallback(itemCallback) {
|
|
189
165
|
return async (collection, contentPaths) => {
|
|
190
166
|
const templateInfo = this.schema.getTemplatesForCollectable(collection);
|
|
191
|
-
await
|
|
192
|
-
const data = await
|
|
167
|
+
await sequential(contentPaths, async (path) => {
|
|
168
|
+
const data = await transformDocumentIntoPayload(
|
|
193
169
|
`${collection.path}/${path}`,
|
|
194
|
-
|
|
170
|
+
transformDocument(
|
|
195
171
|
path,
|
|
196
|
-
await
|
|
172
|
+
await loadAndParseWithAliases(
|
|
197
173
|
this.bridge,
|
|
198
174
|
path,
|
|
199
175
|
collection,
|
|
@@ -215,7 +191,6 @@ var SearchIndexer = class {
|
|
|
215
191
|
};
|
|
216
192
|
}
|
|
217
193
|
async indexContentByPaths(documentPaths) {
|
|
218
|
-
var _a, _b, _c, _d;
|
|
219
194
|
let batch = [];
|
|
220
195
|
const itemCallback = async (item) => {
|
|
221
196
|
batch.push(item);
|
|
@@ -224,8 +199,8 @@ var SearchIndexer = class {
|
|
|
224
199
|
batch = [];
|
|
225
200
|
}
|
|
226
201
|
};
|
|
227
|
-
await
|
|
228
|
-
await
|
|
202
|
+
await this.client.onStartIndexing?.();
|
|
203
|
+
await scanContentByPaths(
|
|
229
204
|
this.schema,
|
|
230
205
|
documentPaths,
|
|
231
206
|
this.makeIndexerCallback(itemCallback)
|
|
@@ -233,11 +208,10 @@ var SearchIndexer = class {
|
|
|
233
208
|
if (batch.length > 0) {
|
|
234
209
|
await this.client.put(batch);
|
|
235
210
|
}
|
|
236
|
-
await
|
|
211
|
+
await this.client.onFinishIndexing?.();
|
|
237
212
|
}
|
|
238
213
|
async indexAllContent() {
|
|
239
|
-
|
|
240
|
-
await ((_b = (_a = this.client).onStartIndexing) == null ? void 0 : _b.call(_a));
|
|
214
|
+
await this.client.onStartIndexing?.();
|
|
241
215
|
let batch = [];
|
|
242
216
|
const itemCallback = async (item) => {
|
|
243
217
|
batch.push(item);
|
|
@@ -246,7 +220,7 @@ var SearchIndexer = class {
|
|
|
246
220
|
batch = [];
|
|
247
221
|
}
|
|
248
222
|
};
|
|
249
|
-
const warnings = await
|
|
223
|
+
const warnings = await scanAllContent(
|
|
250
224
|
this.schema,
|
|
251
225
|
this.bridge,
|
|
252
226
|
this.makeIndexerCallback(itemCallback)
|
|
@@ -254,31 +228,35 @@ var SearchIndexer = class {
|
|
|
254
228
|
if (batch.length > 0) {
|
|
255
229
|
await this.client.put(batch);
|
|
256
230
|
}
|
|
257
|
-
await
|
|
231
|
+
await this.client.onFinishIndexing?.();
|
|
258
232
|
return { warnings };
|
|
259
233
|
}
|
|
260
234
|
async deleteIndexContent(documentPaths) {
|
|
261
|
-
|
|
262
|
-
await ((_b = (_a = this.client).onStartIndexing) == null ? void 0 : _b.call(_a));
|
|
235
|
+
await this.client.onStartIndexing?.();
|
|
263
236
|
await this.client.del(documentPaths);
|
|
264
|
-
await
|
|
237
|
+
await this.client.onFinishIndexing?.();
|
|
265
238
|
}
|
|
266
239
|
};
|
|
267
240
|
|
|
268
241
|
// src/client/index.ts
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
242
|
+
import sqliteLevel from "sqlite-level";
|
|
243
|
+
import si from "search-index";
|
|
244
|
+
import { MemoryLevel } from "memory-level";
|
|
245
|
+
import * as zlib from "node:zlib";
|
|
246
|
+
var { SqliteLevel } = sqliteLevel;
|
|
273
247
|
var DEFAULT_TOKEN_SPLIT_REGEX = /[\p{L}\d_]+/gu;
|
|
274
248
|
var LocalSearchIndexClient = class {
|
|
249
|
+
searchIndex;
|
|
250
|
+
memoryLevel;
|
|
251
|
+
stopwords;
|
|
252
|
+
tokenSplitRegex;
|
|
275
253
|
constructor(options) {
|
|
276
|
-
this.memoryLevel = new
|
|
254
|
+
this.memoryLevel = new MemoryLevel();
|
|
277
255
|
this.stopwords = lookupStopwords(options.stopwordLanguages);
|
|
278
256
|
this.tokenSplitRegex = options.tokenSplitRegex ? new RegExp(options.tokenSplitRegex, "gu") : DEFAULT_TOKEN_SPLIT_REGEX;
|
|
279
257
|
}
|
|
280
258
|
async onStartIndexing() {
|
|
281
|
-
this.searchIndex = await (
|
|
259
|
+
this.searchIndex = await si({
|
|
282
260
|
// @ts-ignore
|
|
283
261
|
db: this.memoryLevel,
|
|
284
262
|
stopwords: this.stopwords,
|
|
@@ -306,15 +284,18 @@ var LocalSearchIndexClient = class {
|
|
|
306
284
|
});
|
|
307
285
|
}
|
|
308
286
|
async export(filename) {
|
|
309
|
-
const
|
|
287
|
+
const sqliteLevel2 = new SqliteLevel({ filename });
|
|
310
288
|
const iterator = this.memoryLevel.iterator();
|
|
311
289
|
for await (const [key, value] of iterator) {
|
|
312
|
-
await
|
|
290
|
+
await sqliteLevel2.put(key, value);
|
|
313
291
|
}
|
|
314
|
-
await
|
|
292
|
+
await sqliteLevel2.close();
|
|
315
293
|
}
|
|
316
294
|
};
|
|
317
295
|
var TinaCMSSearchIndexClient = class extends LocalSearchIndexClient {
|
|
296
|
+
apiUrl;
|
|
297
|
+
branch;
|
|
298
|
+
indexerToken;
|
|
318
299
|
constructor(options) {
|
|
319
300
|
super(options);
|
|
320
301
|
this.apiUrl = options.apiUrl;
|
|
@@ -337,20 +318,21 @@ var TinaCMSSearchIndexClient = class extends LocalSearchIndexClient {
|
|
|
337
318
|
console.error("Failed to parse error response", e);
|
|
338
319
|
}
|
|
339
320
|
throw new Error(
|
|
340
|
-
`Failed to get upload url. Status: ${res.status}${
|
|
321
|
+
`Failed to get upload url. Status: ${res.status}${json?.message ? ` - ${json.message}` : ``}`
|
|
341
322
|
);
|
|
342
323
|
}
|
|
343
324
|
const { signedUrl } = await res.json();
|
|
344
|
-
const
|
|
325
|
+
const sqliteLevel2 = new SqliteLevel({ filename: ":memory:" });
|
|
345
326
|
const iterator = this.memoryLevel.iterator();
|
|
346
327
|
for await (const [key, value] of iterator) {
|
|
347
|
-
await
|
|
328
|
+
await sqliteLevel2.put(key, value);
|
|
348
329
|
}
|
|
349
|
-
const buffer =
|
|
350
|
-
await
|
|
330
|
+
const buffer = sqliteLevel2.db.serialize();
|
|
331
|
+
await sqliteLevel2.close();
|
|
332
|
+
const compressedBuffer = zlib.gzipSync(buffer);
|
|
351
333
|
res = await fetch(signedUrl, {
|
|
352
334
|
method: "PUT",
|
|
353
|
-
body:
|
|
335
|
+
body: new Uint8Array(compressedBuffer)
|
|
354
336
|
});
|
|
355
337
|
if (res.status !== 200) {
|
|
356
338
|
throw new Error(
|
|
@@ -360,10 +342,9 @@ ${await res.text()}`
|
|
|
360
342
|
}
|
|
361
343
|
}
|
|
362
344
|
};
|
|
363
|
-
|
|
364
|
-
0 && (module.exports = {
|
|
345
|
+
export {
|
|
365
346
|
LocalSearchIndexClient,
|
|
366
347
|
SearchIndexer,
|
|
367
348
|
TinaCMSSearchIndexClient,
|
|
368
|
-
si
|
|
369
|
-
}
|
|
349
|
+
si2 as si
|
|
350
|
+
};
|
package/package.json
CHANGED
|
@@ -1,21 +1,20 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@tinacms/search",
|
|
3
|
-
"
|
|
3
|
+
"type": "module",
|
|
4
|
+
"version": "0.0.0-e96d36b-20251210035710",
|
|
4
5
|
"main": "dist/index.js",
|
|
5
|
-
"
|
|
6
|
-
"typings": "dist/index.d.ts",
|
|
6
|
+
"types": "dist/index.d.ts",
|
|
7
7
|
"files": [
|
|
8
|
-
"package.json",
|
|
9
8
|
"dist"
|
|
10
9
|
],
|
|
11
10
|
"exports": {
|
|
12
11
|
".": {
|
|
13
|
-
"
|
|
14
|
-
"
|
|
12
|
+
"types": "./dist/index.d.ts",
|
|
13
|
+
"default": "./dist/index.js"
|
|
15
14
|
},
|
|
16
|
-
"./
|
|
17
|
-
"
|
|
18
|
-
"
|
|
15
|
+
"./index-client": {
|
|
16
|
+
"types": "./dist/index-client.d.ts",
|
|
17
|
+
"default": "./dist/index-client.js"
|
|
19
18
|
}
|
|
20
19
|
},
|
|
21
20
|
"license": "SEE LICENSE IN LICENSE",
|
|
@@ -33,8 +32,8 @@
|
|
|
33
32
|
"search-index": "4.0.0",
|
|
34
33
|
"sqlite-level": "^1.2.1",
|
|
35
34
|
"stopword": "^3.1.4",
|
|
36
|
-
"@tinacms/
|
|
37
|
-
"@tinacms/
|
|
35
|
+
"@tinacms/schema-tools": "0.0.0-e96d36b-20251210035710",
|
|
36
|
+
"@tinacms/graphql": "0.0.0-e96d36b-20251210035710"
|
|
38
37
|
},
|
|
39
38
|
"publishConfig": {
|
|
40
39
|
"registry": "https://registry.npmjs.org"
|
|
@@ -53,7 +52,7 @@
|
|
|
53
52
|
"jest-file-snapshot": "^0.7.0",
|
|
54
53
|
"jest-matcher-utils": "^29.7.0",
|
|
55
54
|
"typescript": "^5.7.3",
|
|
56
|
-
"@tinacms/scripts": "1.
|
|
55
|
+
"@tinacms/scripts": "1.4.2"
|
|
57
56
|
},
|
|
58
57
|
"scripts": {
|
|
59
58
|
"types": "pnpm tsc",
|
package/dist/index-client.mjs
DELETED
|
@@ -1,195 +0,0 @@
|
|
|
1
|
-
import * as sw from "stopword";
|
|
2
|
-
class StringBuilder {
|
|
3
|
-
constructor(limit) {
|
|
4
|
-
this.length = 0;
|
|
5
|
-
this.buffer = [];
|
|
6
|
-
this.limit = limit;
|
|
7
|
-
}
|
|
8
|
-
append(str) {
|
|
9
|
-
if (this.length + str.length > this.limit) {
|
|
10
|
-
return true;
|
|
11
|
-
} else {
|
|
12
|
-
this.buffer.push(str);
|
|
13
|
-
this.length += str.length;
|
|
14
|
-
if (this.length > this.limit) {
|
|
15
|
-
return true;
|
|
16
|
-
}
|
|
17
|
-
return false;
|
|
18
|
-
}
|
|
19
|
-
}
|
|
20
|
-
toString() {
|
|
21
|
-
return this.buffer.join(" ");
|
|
22
|
-
}
|
|
23
|
-
}
|
|
24
|
-
const extractText = (data, acc, indexableNodeTypes) => {
|
|
25
|
-
var _a, _b;
|
|
26
|
-
if (data) {
|
|
27
|
-
if (indexableNodeTypes.indexOf(data.type) !== -1 && (data.text || data.value)) {
|
|
28
|
-
const tokens = tokenizeString(data.text || data.value);
|
|
29
|
-
for (const token of tokens) {
|
|
30
|
-
if (acc.append(token)) {
|
|
31
|
-
return;
|
|
32
|
-
}
|
|
33
|
-
}
|
|
34
|
-
}
|
|
35
|
-
(_b = (_a = data.children) == null ? void 0 : _a.forEach) == null ? void 0 : _b.call(
|
|
36
|
-
_a,
|
|
37
|
-
(child) => extractText(child, acc, indexableNodeTypes)
|
|
38
|
-
);
|
|
39
|
-
}
|
|
40
|
-
};
|
|
41
|
-
const relativePath = (path, collection) => {
|
|
42
|
-
return path.replace(/\\/g, "/").replace(collection.path, "").replace(/^\/|\/$/g, "");
|
|
43
|
-
};
|
|
44
|
-
const tokenizeString = (str) => {
|
|
45
|
-
return str.split(/[\s\.,]+/).map((s) => s.toLowerCase()).filter((s) => s);
|
|
46
|
-
};
|
|
47
|
-
const processTextFieldValue = (value, maxLen) => {
|
|
48
|
-
const tokens = tokenizeString(value);
|
|
49
|
-
const builder = new StringBuilder(maxLen);
|
|
50
|
-
for (const part of tokens) {
|
|
51
|
-
if (builder.append(part)) {
|
|
52
|
-
break;
|
|
53
|
-
}
|
|
54
|
-
}
|
|
55
|
-
return builder.toString();
|
|
56
|
-
};
|
|
57
|
-
const processDocumentForIndexing = (data, path, collection, textIndexLength, field) => {
|
|
58
|
-
if (!field) {
|
|
59
|
-
const relPath = relativePath(path, collection);
|
|
60
|
-
data["_id"] = `${collection.name}:${relPath}`;
|
|
61
|
-
data["_relativePath"] = relPath;
|
|
62
|
-
}
|
|
63
|
-
for (const f of (field == null ? void 0 : field.fields) || collection.fields || []) {
|
|
64
|
-
if (!f.searchable) {
|
|
65
|
-
delete data[f.name];
|
|
66
|
-
continue;
|
|
67
|
-
}
|
|
68
|
-
const isList = f.list;
|
|
69
|
-
if (data[f.name]) {
|
|
70
|
-
if (f.type === "object") {
|
|
71
|
-
if (isList) {
|
|
72
|
-
data[f.name] = data[f.name].map(
|
|
73
|
-
(obj) => processDocumentForIndexing(
|
|
74
|
-
obj,
|
|
75
|
-
path,
|
|
76
|
-
collection,
|
|
77
|
-
textIndexLength,
|
|
78
|
-
f
|
|
79
|
-
)
|
|
80
|
-
);
|
|
81
|
-
} else {
|
|
82
|
-
data[f.name] = processDocumentForIndexing(
|
|
83
|
-
data[f.name],
|
|
84
|
-
path,
|
|
85
|
-
collection,
|
|
86
|
-
textIndexLength,
|
|
87
|
-
f
|
|
88
|
-
);
|
|
89
|
-
}
|
|
90
|
-
} else if (f.type === "string") {
|
|
91
|
-
const fieldTextIndexLength = f.maxSearchIndexFieldLength || textIndexLength;
|
|
92
|
-
if (isList) {
|
|
93
|
-
data[f.name] = data[f.name].map(
|
|
94
|
-
(value) => processTextFieldValue(value, fieldTextIndexLength)
|
|
95
|
-
);
|
|
96
|
-
} else {
|
|
97
|
-
data[f.name] = processTextFieldValue(
|
|
98
|
-
data[f.name],
|
|
99
|
-
fieldTextIndexLength
|
|
100
|
-
);
|
|
101
|
-
}
|
|
102
|
-
} else if (f.type === "rich-text") {
|
|
103
|
-
const fieldTextIndexLength = f.maxSearchIndexFieldLength || textIndexLength;
|
|
104
|
-
if (isList) {
|
|
105
|
-
data[f.name] = data[f.name].map((value) => {
|
|
106
|
-
const acc = new StringBuilder(fieldTextIndexLength);
|
|
107
|
-
extractText(value, acc, ["text", "code_block", "html"]);
|
|
108
|
-
return acc.toString();
|
|
109
|
-
});
|
|
110
|
-
} else {
|
|
111
|
-
const acc = new StringBuilder(fieldTextIndexLength);
|
|
112
|
-
extractText(data[f.name], acc, ["text", "code_block", "html"]);
|
|
113
|
-
data[f.name] = acc.toString();
|
|
114
|
-
}
|
|
115
|
-
}
|
|
116
|
-
}
|
|
117
|
-
}
|
|
118
|
-
return data;
|
|
119
|
-
};
|
|
120
|
-
const memo = {};
|
|
121
|
-
const lookupStopwords = (keys, defaultStopWords = sw.eng) => {
|
|
122
|
-
let stopwords = defaultStopWords;
|
|
123
|
-
if (keys) {
|
|
124
|
-
if (memo[keys.join(",")]) {
|
|
125
|
-
return memo[keys.join(",")];
|
|
126
|
-
}
|
|
127
|
-
stopwords = [];
|
|
128
|
-
for (const key of keys) {
|
|
129
|
-
stopwords.push(...sw[key]);
|
|
130
|
-
}
|
|
131
|
-
memo[keys.join(",")] = stopwords;
|
|
132
|
-
}
|
|
133
|
-
return stopwords;
|
|
134
|
-
};
|
|
135
|
-
const queryToSearchIndexQuery = (query, stopwordLanguages) => {
|
|
136
|
-
let q;
|
|
137
|
-
const parts = query.split(" ");
|
|
138
|
-
const stopwords = lookupStopwords(stopwordLanguages);
|
|
139
|
-
if (parts.length === 1) {
|
|
140
|
-
q = { AND: [parts[0]] };
|
|
141
|
-
} else {
|
|
142
|
-
q = {
|
|
143
|
-
AND: parts.filter(
|
|
144
|
-
(part) => part.toLowerCase() !== "and" && stopwords.indexOf(part.toLowerCase()) === -1
|
|
145
|
-
)
|
|
146
|
-
};
|
|
147
|
-
}
|
|
148
|
-
return q;
|
|
149
|
-
};
|
|
150
|
-
const optionsToSearchIndexOptions = (options) => {
|
|
151
|
-
const opt = {};
|
|
152
|
-
if (options == null ? void 0 : options.limit) {
|
|
153
|
-
opt["PAGE"] = {
|
|
154
|
-
SIZE: options.limit,
|
|
155
|
-
NUMBER: (options == null ? void 0 : options.cursor) ? parseInt(options.cursor) : 0
|
|
156
|
-
};
|
|
157
|
-
}
|
|
158
|
-
return opt;
|
|
159
|
-
};
|
|
160
|
-
const parseSearchIndexResponse = (data, options) => {
|
|
161
|
-
const results = data["RESULT"];
|
|
162
|
-
const total = data["RESULT_LENGTH"];
|
|
163
|
-
if ((options == null ? void 0 : options.cursor) && (options == null ? void 0 : options.limit)) {
|
|
164
|
-
const prevCursor = options.cursor === "0" ? null : (parseInt(options.cursor) - 1).toString();
|
|
165
|
-
const nextCursor = total <= (parseInt(options.cursor) + 1) * options.limit ? null : (parseInt(options.cursor) + 1).toString();
|
|
166
|
-
return {
|
|
167
|
-
results,
|
|
168
|
-
total,
|
|
169
|
-
prevCursor,
|
|
170
|
-
nextCursor
|
|
171
|
-
};
|
|
172
|
-
} else if (!(options == null ? void 0 : options.cursor) && (options == null ? void 0 : options.limit)) {
|
|
173
|
-
const prevCursor = null;
|
|
174
|
-
const nextCursor = total <= options.limit ? null : "1";
|
|
175
|
-
return {
|
|
176
|
-
results,
|
|
177
|
-
total,
|
|
178
|
-
prevCursor,
|
|
179
|
-
nextCursor
|
|
180
|
-
};
|
|
181
|
-
} else {
|
|
182
|
-
return {
|
|
183
|
-
results,
|
|
184
|
-
total,
|
|
185
|
-
prevCursor: null,
|
|
186
|
-
nextCursor: null
|
|
187
|
-
};
|
|
188
|
-
}
|
|
189
|
-
};
|
|
190
|
-
export {
|
|
191
|
-
optionsToSearchIndexOptions,
|
|
192
|
-
parseSearchIndexResponse,
|
|
193
|
-
processDocumentForIndexing,
|
|
194
|
-
queryToSearchIndexQuery
|
|
195
|
-
};
|