@tinacms/search 0.0.0-e70425b-20241028042614 → 0.0.0-e96d36b-20251210035710
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index-client.d.ts +3 -1
- package/dist/index-client.js +183 -195
- package/dist/index.js +60 -74
- package/package.json +16 -17
- package/dist/index-client.mjs +0 -195
package/dist/index-client.d.ts
CHANGED
|
@@ -4,7 +4,9 @@ export declare const queryToSearchIndexQuery: (query: string, stopwordLanguages?
|
|
|
4
4
|
export declare const optionsToSearchIndexOptions: (options?: {
|
|
5
5
|
limit?: number;
|
|
6
6
|
cursor?: string;
|
|
7
|
-
}) => {
|
|
7
|
+
}) => {
|
|
8
|
+
PAGE?: {};
|
|
9
|
+
};
|
|
8
10
|
export declare const parseSearchIndexResponse: (data: any, options?: {
|
|
9
11
|
limit?: number;
|
|
10
12
|
cursor?: string;
|
package/dist/index-client.js
CHANGED
|
@@ -1,215 +1,203 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
"
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
}
|
|
16
|
-
}
|
|
17
|
-
}
|
|
18
|
-
n.default = e;
|
|
19
|
-
return Object.freeze(n);
|
|
1
|
+
var __defProp = Object.defineProperty;
|
|
2
|
+
var __defNormalProp = (obj, key, value) => key in obj ? __defProp(obj, key, { enumerable: true, configurable: true, writable: true, value }) : obj[key] = value;
|
|
3
|
+
var __publicField = (obj, key, value) => {
|
|
4
|
+
__defNormalProp(obj, typeof key !== "symbol" ? key + "" : key, value);
|
|
5
|
+
return value;
|
|
6
|
+
};
|
|
7
|
+
import * as sw from "stopword";
|
|
8
|
+
class StringBuilder {
|
|
9
|
+
constructor(limit) {
|
|
10
|
+
__publicField(this, "buffer");
|
|
11
|
+
__publicField(this, "length", 0);
|
|
12
|
+
__publicField(this, "limit");
|
|
13
|
+
this.buffer = [];
|
|
14
|
+
this.limit = limit;
|
|
20
15
|
}
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
this.buffer
|
|
26
|
-
this.
|
|
27
|
-
|
|
28
|
-
append(str) {
|
|
29
|
-
if (this.length + str.length > this.limit) {
|
|
16
|
+
append(str) {
|
|
17
|
+
if (this.length + str.length > this.limit) {
|
|
18
|
+
return true;
|
|
19
|
+
} else {
|
|
20
|
+
this.buffer.push(str);
|
|
21
|
+
this.length += str.length;
|
|
22
|
+
if (this.length > this.limit) {
|
|
30
23
|
return true;
|
|
31
|
-
} else {
|
|
32
|
-
this.buffer.push(str);
|
|
33
|
-
this.length += str.length;
|
|
34
|
-
if (this.length > this.limit) {
|
|
35
|
-
return true;
|
|
36
|
-
}
|
|
37
|
-
return false;
|
|
38
24
|
}
|
|
25
|
+
return false;
|
|
39
26
|
}
|
|
40
|
-
toString() {
|
|
41
|
-
return this.buffer.join(" ");
|
|
42
|
-
}
|
|
43
27
|
}
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
28
|
+
toString() {
|
|
29
|
+
return this.buffer.join(" ");
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
const extractText = (data, acc, indexableNodeTypes) => {
|
|
33
|
+
var _a, _b;
|
|
34
|
+
if (data) {
|
|
35
|
+
if (indexableNodeTypes.indexOf(data.type) !== -1 && (data.text || data.value)) {
|
|
36
|
+
const tokens = tokenizeString(data.text || data.value);
|
|
37
|
+
for (const token of tokens) {
|
|
38
|
+
if (acc.append(token)) {
|
|
39
|
+
return;
|
|
53
40
|
}
|
|
54
41
|
}
|
|
55
|
-
(_b = (_a = data.children) == null ? void 0 : _a.forEach) == null ? void 0 : _b.call(
|
|
56
|
-
_a,
|
|
57
|
-
(child) => extractText(child, acc, indexableNodeTypes)
|
|
58
|
-
);
|
|
59
42
|
}
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
43
|
+
(_b = (_a = data.children) == null ? void 0 : _a.forEach) == null ? void 0 : _b.call(
|
|
44
|
+
_a,
|
|
45
|
+
(child) => extractText(child, acc, indexableNodeTypes)
|
|
46
|
+
);
|
|
47
|
+
}
|
|
48
|
+
};
|
|
49
|
+
const relativePath = (path, collection) => {
|
|
50
|
+
return path.replace(/\\/g, "/").replace(collection.path, "").replace(/^\/|\/$/g, "");
|
|
51
|
+
};
|
|
52
|
+
const tokenizeString = (str) => {
|
|
53
|
+
return str.split(/[\s\.,]+/).map((s) => s.toLowerCase()).filter((s) => s);
|
|
54
|
+
};
|
|
55
|
+
const processTextFieldValue = (value, maxLen) => {
|
|
56
|
+
const tokens = tokenizeString(value);
|
|
57
|
+
const builder = new StringBuilder(maxLen);
|
|
58
|
+
for (const part of tokens) {
|
|
59
|
+
if (builder.append(part)) {
|
|
60
|
+
break;
|
|
74
61
|
}
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
62
|
+
}
|
|
63
|
+
return builder.toString();
|
|
64
|
+
};
|
|
65
|
+
const processDocumentForIndexing = (data, path, collection, textIndexLength, field) => {
|
|
66
|
+
if (!field) {
|
|
67
|
+
const relPath = relativePath(path, collection);
|
|
68
|
+
data["_id"] = `${collection.name}:${relPath}`;
|
|
69
|
+
data["_relativePath"] = relPath;
|
|
70
|
+
}
|
|
71
|
+
for (const f of (field == null ? void 0 : field.fields) || collection.fields || []) {
|
|
72
|
+
if (!f.searchable) {
|
|
73
|
+
delete data[f.name];
|
|
74
|
+
continue;
|
|
82
75
|
}
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
if (f.type === "object") {
|
|
91
|
-
if (isList) {
|
|
92
|
-
data[f.name] = data[f.name].map(
|
|
93
|
-
(obj) => processDocumentForIndexing(
|
|
94
|
-
obj,
|
|
95
|
-
path,
|
|
96
|
-
collection,
|
|
97
|
-
textIndexLength,
|
|
98
|
-
f
|
|
99
|
-
)
|
|
100
|
-
);
|
|
101
|
-
} else {
|
|
102
|
-
data[f.name] = processDocumentForIndexing(
|
|
103
|
-
data[f.name],
|
|
76
|
+
const isList = f.list;
|
|
77
|
+
if (data[f.name]) {
|
|
78
|
+
if (f.type === "object") {
|
|
79
|
+
if (isList) {
|
|
80
|
+
data[f.name] = data[f.name].map(
|
|
81
|
+
(obj) => processDocumentForIndexing(
|
|
82
|
+
obj,
|
|
104
83
|
path,
|
|
105
84
|
collection,
|
|
106
85
|
textIndexLength,
|
|
107
86
|
f
|
|
108
|
-
)
|
|
109
|
-
|
|
110
|
-
} else
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
87
|
+
)
|
|
88
|
+
);
|
|
89
|
+
} else {
|
|
90
|
+
data[f.name] = processDocumentForIndexing(
|
|
91
|
+
data[f.name],
|
|
92
|
+
path,
|
|
93
|
+
collection,
|
|
94
|
+
textIndexLength,
|
|
95
|
+
f
|
|
96
|
+
);
|
|
97
|
+
}
|
|
98
|
+
} else if (f.type === "string") {
|
|
99
|
+
const fieldTextIndexLength = f.maxSearchIndexFieldLength || textIndexLength;
|
|
100
|
+
if (isList) {
|
|
101
|
+
data[f.name] = data[f.name].map(
|
|
102
|
+
(value) => processTextFieldValue(value, fieldTextIndexLength)
|
|
103
|
+
);
|
|
104
|
+
} else {
|
|
105
|
+
data[f.name] = processTextFieldValue(
|
|
106
|
+
data[f.name],
|
|
107
|
+
fieldTextIndexLength
|
|
108
|
+
);
|
|
109
|
+
}
|
|
110
|
+
} else if (f.type === "rich-text") {
|
|
111
|
+
const fieldTextIndexLength = f.maxSearchIndexFieldLength || textIndexLength;
|
|
112
|
+
if (isList) {
|
|
113
|
+
data[f.name] = data[f.name].map((value) => {
|
|
131
114
|
const acc = new StringBuilder(fieldTextIndexLength);
|
|
132
|
-
extractText(
|
|
133
|
-
|
|
134
|
-
}
|
|
115
|
+
extractText(value, acc, ["text", "code_block", "html"]);
|
|
116
|
+
return acc.toString();
|
|
117
|
+
});
|
|
118
|
+
} else {
|
|
119
|
+
const acc = new StringBuilder(fieldTextIndexLength);
|
|
120
|
+
extractText(data[f.name], acc, ["text", "code_block", "html"]);
|
|
121
|
+
data[f.name] = acc.toString();
|
|
135
122
|
}
|
|
136
123
|
}
|
|
137
124
|
}
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
stopwords = [];
|
|
148
|
-
for (const key of keys) {
|
|
149
|
-
stopwords.push(...sw__namespace[key]);
|
|
150
|
-
}
|
|
151
|
-
memo[keys.join(",")] = stopwords;
|
|
152
|
-
}
|
|
153
|
-
return stopwords;
|
|
154
|
-
};
|
|
155
|
-
const queryToSearchIndexQuery = (query, stopwordLanguages) => {
|
|
156
|
-
let q;
|
|
157
|
-
const parts = query.split(" ");
|
|
158
|
-
const stopwords = lookupStopwords(stopwordLanguages);
|
|
159
|
-
if (parts.length === 1) {
|
|
160
|
-
q = { AND: [parts[0]] };
|
|
161
|
-
} else {
|
|
162
|
-
q = {
|
|
163
|
-
AND: parts.filter(
|
|
164
|
-
(part) => part.toLowerCase() !== "and" && stopwords.indexOf(part.toLowerCase()) === -1
|
|
165
|
-
)
|
|
166
|
-
};
|
|
167
|
-
}
|
|
168
|
-
return q;
|
|
169
|
-
};
|
|
170
|
-
const optionsToSearchIndexOptions = (options) => {
|
|
171
|
-
const opt = {};
|
|
172
|
-
if (options == null ? void 0 : options.limit) {
|
|
173
|
-
opt["PAGE"] = {
|
|
174
|
-
SIZE: options.limit,
|
|
175
|
-
NUMBER: (options == null ? void 0 : options.cursor) ? parseInt(options.cursor) : 0
|
|
176
|
-
};
|
|
125
|
+
}
|
|
126
|
+
return data;
|
|
127
|
+
};
|
|
128
|
+
const memo = {};
|
|
129
|
+
const lookupStopwords = (keys, defaultStopWords = sw.eng) => {
|
|
130
|
+
let stopwords = defaultStopWords;
|
|
131
|
+
if (keys) {
|
|
132
|
+
if (memo[keys.join(",")]) {
|
|
133
|
+
return memo[keys.join(",")];
|
|
177
134
|
}
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
const results = data["RESULT"];
|
|
182
|
-
const total = data["RESULT_LENGTH"];
|
|
183
|
-
if ((options == null ? void 0 : options.cursor) && (options == null ? void 0 : options.limit)) {
|
|
184
|
-
const prevCursor = options.cursor === "0" ? null : (parseInt(options.cursor) - 1).toString();
|
|
185
|
-
const nextCursor = total <= (parseInt(options.cursor) + 1) * options.limit ? null : (parseInt(options.cursor) + 1).toString();
|
|
186
|
-
return {
|
|
187
|
-
results,
|
|
188
|
-
total,
|
|
189
|
-
prevCursor,
|
|
190
|
-
nextCursor
|
|
191
|
-
};
|
|
192
|
-
} else if (!(options == null ? void 0 : options.cursor) && (options == null ? void 0 : options.limit)) {
|
|
193
|
-
const prevCursor = null;
|
|
194
|
-
const nextCursor = total <= options.limit ? null : "1";
|
|
195
|
-
return {
|
|
196
|
-
results,
|
|
197
|
-
total,
|
|
198
|
-
prevCursor,
|
|
199
|
-
nextCursor
|
|
200
|
-
};
|
|
201
|
-
} else {
|
|
202
|
-
return {
|
|
203
|
-
results,
|
|
204
|
-
total,
|
|
205
|
-
prevCursor: null,
|
|
206
|
-
nextCursor: null
|
|
207
|
-
};
|
|
135
|
+
stopwords = [];
|
|
136
|
+
for (const key of keys) {
|
|
137
|
+
stopwords.push(...sw[key]);
|
|
208
138
|
}
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
139
|
+
memo[keys.join(",")] = stopwords;
|
|
140
|
+
}
|
|
141
|
+
return stopwords;
|
|
142
|
+
};
|
|
143
|
+
const queryToSearchIndexQuery = (query, stopwordLanguages) => {
|
|
144
|
+
let q;
|
|
145
|
+
const parts = query.split(" ");
|
|
146
|
+
const stopwords = lookupStopwords(stopwordLanguages);
|
|
147
|
+
if (parts.length === 1) {
|
|
148
|
+
q = { AND: [parts[0]] };
|
|
149
|
+
} else {
|
|
150
|
+
q = {
|
|
151
|
+
AND: parts.filter(
|
|
152
|
+
(part) => part.toLowerCase() !== "and" && stopwords.indexOf(part.toLowerCase()) === -1
|
|
153
|
+
)
|
|
154
|
+
};
|
|
155
|
+
}
|
|
156
|
+
return q;
|
|
157
|
+
};
|
|
158
|
+
const optionsToSearchIndexOptions = (options) => {
|
|
159
|
+
const opt = {};
|
|
160
|
+
if (options == null ? void 0 : options.limit) {
|
|
161
|
+
opt["PAGE"] = {
|
|
162
|
+
SIZE: options.limit,
|
|
163
|
+
NUMBER: (options == null ? void 0 : options.cursor) ? parseInt(options.cursor) : 0
|
|
164
|
+
};
|
|
165
|
+
}
|
|
166
|
+
return opt;
|
|
167
|
+
};
|
|
168
|
+
const parseSearchIndexResponse = (data, options) => {
|
|
169
|
+
const results = data["RESULT"];
|
|
170
|
+
const total = data["RESULT_LENGTH"];
|
|
171
|
+
if ((options == null ? void 0 : options.cursor) && (options == null ? void 0 : options.limit)) {
|
|
172
|
+
const prevCursor = options.cursor === "0" ? null : (parseInt(options.cursor) - 1).toString();
|
|
173
|
+
const nextCursor = total <= (parseInt(options.cursor) + 1) * options.limit ? null : (parseInt(options.cursor) + 1).toString();
|
|
174
|
+
return {
|
|
175
|
+
results,
|
|
176
|
+
total,
|
|
177
|
+
prevCursor,
|
|
178
|
+
nextCursor
|
|
179
|
+
};
|
|
180
|
+
} else if (!(options == null ? void 0 : options.cursor) && (options == null ? void 0 : options.limit)) {
|
|
181
|
+
const prevCursor = null;
|
|
182
|
+
const nextCursor = total <= options.limit ? null : "1";
|
|
183
|
+
return {
|
|
184
|
+
results,
|
|
185
|
+
total,
|
|
186
|
+
prevCursor,
|
|
187
|
+
nextCursor
|
|
188
|
+
};
|
|
189
|
+
} else {
|
|
190
|
+
return {
|
|
191
|
+
results,
|
|
192
|
+
total,
|
|
193
|
+
prevCursor: null,
|
|
194
|
+
nextCursor: null
|
|
195
|
+
};
|
|
196
|
+
}
|
|
197
|
+
};
|
|
198
|
+
export {
|
|
199
|
+
optionsToSearchIndexOptions,
|
|
200
|
+
parseSearchIndexResponse,
|
|
201
|
+
processDocumentForIndexing,
|
|
202
|
+
queryToSearchIndexQuery
|
|
203
|
+
};
|
package/dist/index.js
CHANGED
|
@@ -1,46 +1,23 @@
|
|
|
1
|
-
var __create = Object.create;
|
|
2
|
-
var __defProp = Object.defineProperty;
|
|
3
|
-
var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
|
|
4
|
-
var __getOwnPropNames = Object.getOwnPropertyNames;
|
|
5
|
-
var __getProtoOf = Object.getPrototypeOf;
|
|
6
|
-
var __hasOwnProp = Object.prototype.hasOwnProperty;
|
|
7
|
-
var __export = (target, all) => {
|
|
8
|
-
for (var name in all)
|
|
9
|
-
__defProp(target, name, { get: all[name], enumerable: true });
|
|
10
|
-
};
|
|
11
|
-
var __copyProps = (to, from, except, desc) => {
|
|
12
|
-
if (from && typeof from === "object" || typeof from === "function") {
|
|
13
|
-
for (let key of __getOwnPropNames(from))
|
|
14
|
-
if (!__hasOwnProp.call(to, key) && key !== except)
|
|
15
|
-
__defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
|
|
16
|
-
}
|
|
17
|
-
return to;
|
|
18
|
-
};
|
|
19
|
-
var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
|
|
20
|
-
isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
|
|
21
|
-
mod
|
|
22
|
-
));
|
|
23
|
-
var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
|
|
24
|
-
|
|
25
1
|
// src/index.ts
|
|
26
|
-
|
|
27
|
-
__export(src_exports, {
|
|
28
|
-
LocalSearchIndexClient: () => LocalSearchIndexClient,
|
|
29
|
-
SearchIndexer: () => SearchIndexer,
|
|
30
|
-
TinaCMSSearchIndexClient: () => TinaCMSSearchIndexClient,
|
|
31
|
-
si: () => import_search_index2.default
|
|
32
|
-
});
|
|
33
|
-
module.exports = __toCommonJS(src_exports);
|
|
34
|
-
var import_search_index2 = __toESM(require("search-index"));
|
|
2
|
+
import si2 from "search-index";
|
|
35
3
|
|
|
36
4
|
// src/indexer/index.ts
|
|
37
|
-
|
|
5
|
+
import {
|
|
6
|
+
loadAndParseWithAliases,
|
|
7
|
+
sequential,
|
|
8
|
+
scanAllContent,
|
|
9
|
+
scanContentByPaths,
|
|
10
|
+
transformDocument,
|
|
11
|
+
transformDocumentIntoPayload
|
|
12
|
+
} from "@tinacms/graphql";
|
|
38
13
|
|
|
39
14
|
// src/indexer/utils.ts
|
|
40
|
-
|
|
15
|
+
import * as sw from "stopword";
|
|
41
16
|
var StringBuilder = class {
|
|
17
|
+
buffer;
|
|
18
|
+
length = 0;
|
|
19
|
+
limit;
|
|
42
20
|
constructor(limit) {
|
|
43
|
-
this.length = 0;
|
|
44
21
|
this.buffer = [];
|
|
45
22
|
this.limit = limit;
|
|
46
23
|
}
|
|
@@ -61,7 +38,6 @@ var StringBuilder = class {
|
|
|
61
38
|
}
|
|
62
39
|
};
|
|
63
40
|
var extractText = (data, acc, indexableNodeTypes) => {
|
|
64
|
-
var _a, _b;
|
|
65
41
|
if (data) {
|
|
66
42
|
if (indexableNodeTypes.indexOf(data.type) !== -1 && (data.text || data.value)) {
|
|
67
43
|
const tokens = tokenizeString(data.text || data.value);
|
|
@@ -71,8 +47,7 @@ var extractText = (data, acc, indexableNodeTypes) => {
|
|
|
71
47
|
}
|
|
72
48
|
}
|
|
73
49
|
}
|
|
74
|
-
|
|
75
|
-
_a,
|
|
50
|
+
data.children?.forEach?.(
|
|
76
51
|
(child) => extractText(child, acc, indexableNodeTypes)
|
|
77
52
|
);
|
|
78
53
|
}
|
|
@@ -99,7 +74,7 @@ var processDocumentForIndexing = (data, path, collection, textIndexLength, field
|
|
|
99
74
|
data["_id"] = `${collection.name}:${relPath}`;
|
|
100
75
|
data["_relativePath"] = relPath;
|
|
101
76
|
}
|
|
102
|
-
for (const f of
|
|
77
|
+
for (const f of field?.fields || collection.fields || []) {
|
|
103
78
|
if (!f.searchable) {
|
|
104
79
|
delete data[f.name];
|
|
105
80
|
continue;
|
|
@@ -174,6 +149,11 @@ var lookupStopwords = (keys, defaultStopWords = sw.eng) => {
|
|
|
174
149
|
|
|
175
150
|
// src/indexer/index.ts
|
|
176
151
|
var SearchIndexer = class {
|
|
152
|
+
batchSize;
|
|
153
|
+
client;
|
|
154
|
+
bridge;
|
|
155
|
+
schema;
|
|
156
|
+
textIndexLength;
|
|
177
157
|
constructor(options) {
|
|
178
158
|
this.client = options.client;
|
|
179
159
|
this.bridge = options.bridge;
|
|
@@ -184,12 +164,12 @@ var SearchIndexer = class {
|
|
|
184
164
|
makeIndexerCallback(itemCallback) {
|
|
185
165
|
return async (collection, contentPaths) => {
|
|
186
166
|
const templateInfo = this.schema.getTemplatesForCollectable(collection);
|
|
187
|
-
await
|
|
188
|
-
const data = await
|
|
167
|
+
await sequential(contentPaths, async (path) => {
|
|
168
|
+
const data = await transformDocumentIntoPayload(
|
|
189
169
|
`${collection.path}/${path}`,
|
|
190
|
-
|
|
170
|
+
transformDocument(
|
|
191
171
|
path,
|
|
192
|
-
await
|
|
172
|
+
await loadAndParseWithAliases(
|
|
193
173
|
this.bridge,
|
|
194
174
|
path,
|
|
195
175
|
collection,
|
|
@@ -211,7 +191,6 @@ var SearchIndexer = class {
|
|
|
211
191
|
};
|
|
212
192
|
}
|
|
213
193
|
async indexContentByPaths(documentPaths) {
|
|
214
|
-
var _a, _b, _c, _d;
|
|
215
194
|
let batch = [];
|
|
216
195
|
const itemCallback = async (item) => {
|
|
217
196
|
batch.push(item);
|
|
@@ -220,8 +199,8 @@ var SearchIndexer = class {
|
|
|
220
199
|
batch = [];
|
|
221
200
|
}
|
|
222
201
|
};
|
|
223
|
-
await
|
|
224
|
-
await
|
|
202
|
+
await this.client.onStartIndexing?.();
|
|
203
|
+
await scanContentByPaths(
|
|
225
204
|
this.schema,
|
|
226
205
|
documentPaths,
|
|
227
206
|
this.makeIndexerCallback(itemCallback)
|
|
@@ -229,11 +208,10 @@ var SearchIndexer = class {
|
|
|
229
208
|
if (batch.length > 0) {
|
|
230
209
|
await this.client.put(batch);
|
|
231
210
|
}
|
|
232
|
-
await
|
|
211
|
+
await this.client.onFinishIndexing?.();
|
|
233
212
|
}
|
|
234
213
|
async indexAllContent() {
|
|
235
|
-
|
|
236
|
-
await ((_b = (_a = this.client).onStartIndexing) == null ? void 0 : _b.call(_a));
|
|
214
|
+
await this.client.onStartIndexing?.();
|
|
237
215
|
let batch = [];
|
|
238
216
|
const itemCallback = async (item) => {
|
|
239
217
|
batch.push(item);
|
|
@@ -242,7 +220,7 @@ var SearchIndexer = class {
|
|
|
242
220
|
batch = [];
|
|
243
221
|
}
|
|
244
222
|
};
|
|
245
|
-
const warnings = await
|
|
223
|
+
const warnings = await scanAllContent(
|
|
246
224
|
this.schema,
|
|
247
225
|
this.bridge,
|
|
248
226
|
this.makeIndexerCallback(itemCallback)
|
|
@@ -250,31 +228,36 @@ var SearchIndexer = class {
|
|
|
250
228
|
if (batch.length > 0) {
|
|
251
229
|
await this.client.put(batch);
|
|
252
230
|
}
|
|
253
|
-
await
|
|
231
|
+
await this.client.onFinishIndexing?.();
|
|
254
232
|
return { warnings };
|
|
255
233
|
}
|
|
256
234
|
async deleteIndexContent(documentPaths) {
|
|
257
|
-
|
|
258
|
-
await ((_b = (_a = this.client).onStartIndexing) == null ? void 0 : _b.call(_a));
|
|
235
|
+
await this.client.onStartIndexing?.();
|
|
259
236
|
await this.client.del(documentPaths);
|
|
260
|
-
await
|
|
237
|
+
await this.client.onFinishIndexing?.();
|
|
261
238
|
}
|
|
262
239
|
};
|
|
263
240
|
|
|
264
241
|
// src/client/index.ts
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
242
|
+
import sqliteLevel from "sqlite-level";
|
|
243
|
+
import si from "search-index";
|
|
244
|
+
import { MemoryLevel } from "memory-level";
|
|
245
|
+
import * as zlib from "node:zlib";
|
|
246
|
+
var { SqliteLevel } = sqliteLevel;
|
|
269
247
|
var DEFAULT_TOKEN_SPLIT_REGEX = /[\p{L}\d_]+/gu;
|
|
270
248
|
var LocalSearchIndexClient = class {
|
|
249
|
+
searchIndex;
|
|
250
|
+
memoryLevel;
|
|
251
|
+
stopwords;
|
|
252
|
+
tokenSplitRegex;
|
|
271
253
|
constructor(options) {
|
|
272
|
-
this.memoryLevel = new
|
|
254
|
+
this.memoryLevel = new MemoryLevel();
|
|
273
255
|
this.stopwords = lookupStopwords(options.stopwordLanguages);
|
|
274
256
|
this.tokenSplitRegex = options.tokenSplitRegex ? new RegExp(options.tokenSplitRegex, "gu") : DEFAULT_TOKEN_SPLIT_REGEX;
|
|
275
257
|
}
|
|
276
258
|
async onStartIndexing() {
|
|
277
|
-
this.searchIndex = await (
|
|
259
|
+
this.searchIndex = await si({
|
|
260
|
+
// @ts-ignore
|
|
278
261
|
db: this.memoryLevel,
|
|
279
262
|
stopwords: this.stopwords,
|
|
280
263
|
tokenSplitRegex: this.tokenSplitRegex
|
|
@@ -301,15 +284,18 @@ var LocalSearchIndexClient = class {
|
|
|
301
284
|
});
|
|
302
285
|
}
|
|
303
286
|
async export(filename) {
|
|
304
|
-
const
|
|
287
|
+
const sqliteLevel2 = new SqliteLevel({ filename });
|
|
305
288
|
const iterator = this.memoryLevel.iterator();
|
|
306
289
|
for await (const [key, value] of iterator) {
|
|
307
|
-
await
|
|
290
|
+
await sqliteLevel2.put(key, value);
|
|
308
291
|
}
|
|
309
|
-
await
|
|
292
|
+
await sqliteLevel2.close();
|
|
310
293
|
}
|
|
311
294
|
};
|
|
312
295
|
var TinaCMSSearchIndexClient = class extends LocalSearchIndexClient {
|
|
296
|
+
apiUrl;
|
|
297
|
+
branch;
|
|
298
|
+
indexerToken;
|
|
313
299
|
constructor(options) {
|
|
314
300
|
super(options);
|
|
315
301
|
this.apiUrl = options.apiUrl;
|
|
@@ -332,20 +318,21 @@ var TinaCMSSearchIndexClient = class extends LocalSearchIndexClient {
|
|
|
332
318
|
console.error("Failed to parse error response", e);
|
|
333
319
|
}
|
|
334
320
|
throw new Error(
|
|
335
|
-
`Failed to get upload url. Status: ${res.status}${
|
|
321
|
+
`Failed to get upload url. Status: ${res.status}${json?.message ? ` - ${json.message}` : ``}`
|
|
336
322
|
);
|
|
337
323
|
}
|
|
338
324
|
const { signedUrl } = await res.json();
|
|
339
|
-
const
|
|
325
|
+
const sqliteLevel2 = new SqliteLevel({ filename: ":memory:" });
|
|
340
326
|
const iterator = this.memoryLevel.iterator();
|
|
341
327
|
for await (const [key, value] of iterator) {
|
|
342
|
-
await
|
|
328
|
+
await sqliteLevel2.put(key, value);
|
|
343
329
|
}
|
|
344
|
-
const buffer =
|
|
345
|
-
await
|
|
330
|
+
const buffer = sqliteLevel2.db.serialize();
|
|
331
|
+
await sqliteLevel2.close();
|
|
332
|
+
const compressedBuffer = zlib.gzipSync(buffer);
|
|
346
333
|
res = await fetch(signedUrl, {
|
|
347
334
|
method: "PUT",
|
|
348
|
-
body:
|
|
335
|
+
body: new Uint8Array(compressedBuffer)
|
|
349
336
|
});
|
|
350
337
|
if (res.status !== 200) {
|
|
351
338
|
throw new Error(
|
|
@@ -355,10 +342,9 @@ ${await res.text()}`
|
|
|
355
342
|
}
|
|
356
343
|
}
|
|
357
344
|
};
|
|
358
|
-
|
|
359
|
-
0 && (module.exports = {
|
|
345
|
+
export {
|
|
360
346
|
LocalSearchIndexClient,
|
|
361
347
|
SearchIndexer,
|
|
362
348
|
TinaCMSSearchIndexClient,
|
|
363
|
-
si
|
|
364
|
-
}
|
|
349
|
+
si2 as si
|
|
350
|
+
};
|
package/package.json
CHANGED
|
@@ -1,21 +1,20 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@tinacms/search",
|
|
3
|
-
"
|
|
3
|
+
"type": "module",
|
|
4
|
+
"version": "0.0.0-e96d36b-20251210035710",
|
|
4
5
|
"main": "dist/index.js",
|
|
5
|
-
"
|
|
6
|
-
"typings": "dist/index.d.ts",
|
|
6
|
+
"types": "dist/index.d.ts",
|
|
7
7
|
"files": [
|
|
8
|
-
"package.json",
|
|
9
8
|
"dist"
|
|
10
9
|
],
|
|
11
10
|
"exports": {
|
|
12
11
|
".": {
|
|
13
|
-
"
|
|
14
|
-
"
|
|
12
|
+
"types": "./dist/index.d.ts",
|
|
13
|
+
"default": "./dist/index.js"
|
|
15
14
|
},
|
|
16
|
-
"./
|
|
17
|
-
"
|
|
18
|
-
"
|
|
15
|
+
"./index-client": {
|
|
16
|
+
"types": "./dist/index-client.d.ts",
|
|
17
|
+
"default": "./dist/index-client.js"
|
|
19
18
|
}
|
|
20
19
|
},
|
|
21
20
|
"license": "SEE LICENSE IN LICENSE",
|
|
@@ -31,10 +30,10 @@
|
|
|
31
30
|
"dependencies": {
|
|
32
31
|
"memory-level": "^1.0.0",
|
|
33
32
|
"search-index": "4.0.0",
|
|
34
|
-
"sqlite-level": "^1.2.
|
|
35
|
-
"stopword": "^3.1.
|
|
36
|
-
"@tinacms/
|
|
37
|
-
"@tinacms/
|
|
33
|
+
"sqlite-level": "^1.2.1",
|
|
34
|
+
"stopword": "^3.1.4",
|
|
35
|
+
"@tinacms/schema-tools": "0.0.0-e96d36b-20251210035710",
|
|
36
|
+
"@tinacms/graphql": "0.0.0-e96d36b-20251210035710"
|
|
38
37
|
},
|
|
39
38
|
"publishConfig": {
|
|
40
39
|
"registry": "https://registry.npmjs.org"
|
|
@@ -44,16 +43,16 @@
|
|
|
44
43
|
"directory": "packages/@tinacms/search"
|
|
45
44
|
},
|
|
46
45
|
"devDependencies": {
|
|
47
|
-
"@types/jest": "^29.5.
|
|
46
|
+
"@types/jest": "^29.5.14",
|
|
48
47
|
"@types/micromatch": "^4.0.9",
|
|
49
|
-
"@types/node": "^22.
|
|
48
|
+
"@types/node": "^22.13.1",
|
|
50
49
|
"@types/search-index": "^3.2.4",
|
|
51
50
|
"jest": "^29.7.0",
|
|
52
51
|
"jest-diff": "^29.7.0",
|
|
53
52
|
"jest-file-snapshot": "^0.7.0",
|
|
54
53
|
"jest-matcher-utils": "^29.7.0",
|
|
55
|
-
"typescript": "^5.
|
|
56
|
-
"@tinacms/scripts": "1.
|
|
54
|
+
"typescript": "^5.7.3",
|
|
55
|
+
"@tinacms/scripts": "1.4.2"
|
|
57
56
|
},
|
|
58
57
|
"scripts": {
|
|
59
58
|
"types": "pnpm tsc",
|
package/dist/index-client.mjs
DELETED
|
@@ -1,195 +0,0 @@
|
|
|
1
|
-
import * as sw from "stopword";
|
|
2
|
-
class StringBuilder {
|
|
3
|
-
constructor(limit) {
|
|
4
|
-
this.length = 0;
|
|
5
|
-
this.buffer = [];
|
|
6
|
-
this.limit = limit;
|
|
7
|
-
}
|
|
8
|
-
append(str) {
|
|
9
|
-
if (this.length + str.length > this.limit) {
|
|
10
|
-
return true;
|
|
11
|
-
} else {
|
|
12
|
-
this.buffer.push(str);
|
|
13
|
-
this.length += str.length;
|
|
14
|
-
if (this.length > this.limit) {
|
|
15
|
-
return true;
|
|
16
|
-
}
|
|
17
|
-
return false;
|
|
18
|
-
}
|
|
19
|
-
}
|
|
20
|
-
toString() {
|
|
21
|
-
return this.buffer.join(" ");
|
|
22
|
-
}
|
|
23
|
-
}
|
|
24
|
-
const extractText = (data, acc, indexableNodeTypes) => {
|
|
25
|
-
var _a, _b;
|
|
26
|
-
if (data) {
|
|
27
|
-
if (indexableNodeTypes.indexOf(data.type) !== -1 && (data.text || data.value)) {
|
|
28
|
-
const tokens = tokenizeString(data.text || data.value);
|
|
29
|
-
for (const token of tokens) {
|
|
30
|
-
if (acc.append(token)) {
|
|
31
|
-
return;
|
|
32
|
-
}
|
|
33
|
-
}
|
|
34
|
-
}
|
|
35
|
-
(_b = (_a = data.children) == null ? void 0 : _a.forEach) == null ? void 0 : _b.call(
|
|
36
|
-
_a,
|
|
37
|
-
(child) => extractText(child, acc, indexableNodeTypes)
|
|
38
|
-
);
|
|
39
|
-
}
|
|
40
|
-
};
|
|
41
|
-
const relativePath = (path, collection) => {
|
|
42
|
-
return path.replace(/\\/g, "/").replace(collection.path, "").replace(/^\/|\/$/g, "");
|
|
43
|
-
};
|
|
44
|
-
const tokenizeString = (str) => {
|
|
45
|
-
return str.split(/[\s\.,]+/).map((s) => s.toLowerCase()).filter((s) => s);
|
|
46
|
-
};
|
|
47
|
-
const processTextFieldValue = (value, maxLen) => {
|
|
48
|
-
const tokens = tokenizeString(value);
|
|
49
|
-
const builder = new StringBuilder(maxLen);
|
|
50
|
-
for (const part of tokens) {
|
|
51
|
-
if (builder.append(part)) {
|
|
52
|
-
break;
|
|
53
|
-
}
|
|
54
|
-
}
|
|
55
|
-
return builder.toString();
|
|
56
|
-
};
|
|
57
|
-
const processDocumentForIndexing = (data, path, collection, textIndexLength, field) => {
|
|
58
|
-
if (!field) {
|
|
59
|
-
const relPath = relativePath(path, collection);
|
|
60
|
-
data["_id"] = `${collection.name}:${relPath}`;
|
|
61
|
-
data["_relativePath"] = relPath;
|
|
62
|
-
}
|
|
63
|
-
for (const f of (field == null ? void 0 : field.fields) || collection.fields || []) {
|
|
64
|
-
if (!f.searchable) {
|
|
65
|
-
delete data[f.name];
|
|
66
|
-
continue;
|
|
67
|
-
}
|
|
68
|
-
const isList = f.list;
|
|
69
|
-
if (data[f.name]) {
|
|
70
|
-
if (f.type === "object") {
|
|
71
|
-
if (isList) {
|
|
72
|
-
data[f.name] = data[f.name].map(
|
|
73
|
-
(obj) => processDocumentForIndexing(
|
|
74
|
-
obj,
|
|
75
|
-
path,
|
|
76
|
-
collection,
|
|
77
|
-
textIndexLength,
|
|
78
|
-
f
|
|
79
|
-
)
|
|
80
|
-
);
|
|
81
|
-
} else {
|
|
82
|
-
data[f.name] = processDocumentForIndexing(
|
|
83
|
-
data[f.name],
|
|
84
|
-
path,
|
|
85
|
-
collection,
|
|
86
|
-
textIndexLength,
|
|
87
|
-
f
|
|
88
|
-
);
|
|
89
|
-
}
|
|
90
|
-
} else if (f.type === "string") {
|
|
91
|
-
const fieldTextIndexLength = f.maxSearchIndexFieldLength || textIndexLength;
|
|
92
|
-
if (isList) {
|
|
93
|
-
data[f.name] = data[f.name].map(
|
|
94
|
-
(value) => processTextFieldValue(value, fieldTextIndexLength)
|
|
95
|
-
);
|
|
96
|
-
} else {
|
|
97
|
-
data[f.name] = processTextFieldValue(
|
|
98
|
-
data[f.name],
|
|
99
|
-
fieldTextIndexLength
|
|
100
|
-
);
|
|
101
|
-
}
|
|
102
|
-
} else if (f.type === "rich-text") {
|
|
103
|
-
const fieldTextIndexLength = f.maxSearchIndexFieldLength || textIndexLength;
|
|
104
|
-
if (isList) {
|
|
105
|
-
data[f.name] = data[f.name].map((value) => {
|
|
106
|
-
const acc = new StringBuilder(fieldTextIndexLength);
|
|
107
|
-
extractText(value, acc, ["text", "code_block", "html"]);
|
|
108
|
-
return acc.toString();
|
|
109
|
-
});
|
|
110
|
-
} else {
|
|
111
|
-
const acc = new StringBuilder(fieldTextIndexLength);
|
|
112
|
-
extractText(data[f.name], acc, ["text", "code_block", "html"]);
|
|
113
|
-
data[f.name] = acc.toString();
|
|
114
|
-
}
|
|
115
|
-
}
|
|
116
|
-
}
|
|
117
|
-
}
|
|
118
|
-
return data;
|
|
119
|
-
};
|
|
120
|
-
const memo = {};
|
|
121
|
-
const lookupStopwords = (keys, defaultStopWords = sw.eng) => {
|
|
122
|
-
let stopwords = defaultStopWords;
|
|
123
|
-
if (keys) {
|
|
124
|
-
if (memo[keys.join(",")]) {
|
|
125
|
-
return memo[keys.join(",")];
|
|
126
|
-
}
|
|
127
|
-
stopwords = [];
|
|
128
|
-
for (const key of keys) {
|
|
129
|
-
stopwords.push(...sw[key]);
|
|
130
|
-
}
|
|
131
|
-
memo[keys.join(",")] = stopwords;
|
|
132
|
-
}
|
|
133
|
-
return stopwords;
|
|
134
|
-
};
|
|
135
|
-
const queryToSearchIndexQuery = (query, stopwordLanguages) => {
|
|
136
|
-
let q;
|
|
137
|
-
const parts = query.split(" ");
|
|
138
|
-
const stopwords = lookupStopwords(stopwordLanguages);
|
|
139
|
-
if (parts.length === 1) {
|
|
140
|
-
q = { AND: [parts[0]] };
|
|
141
|
-
} else {
|
|
142
|
-
q = {
|
|
143
|
-
AND: parts.filter(
|
|
144
|
-
(part) => part.toLowerCase() !== "and" && stopwords.indexOf(part.toLowerCase()) === -1
|
|
145
|
-
)
|
|
146
|
-
};
|
|
147
|
-
}
|
|
148
|
-
return q;
|
|
149
|
-
};
|
|
150
|
-
const optionsToSearchIndexOptions = (options) => {
|
|
151
|
-
const opt = {};
|
|
152
|
-
if (options == null ? void 0 : options.limit) {
|
|
153
|
-
opt["PAGE"] = {
|
|
154
|
-
SIZE: options.limit,
|
|
155
|
-
NUMBER: (options == null ? void 0 : options.cursor) ? parseInt(options.cursor) : 0
|
|
156
|
-
};
|
|
157
|
-
}
|
|
158
|
-
return opt;
|
|
159
|
-
};
|
|
160
|
-
const parseSearchIndexResponse = (data, options) => {
|
|
161
|
-
const results = data["RESULT"];
|
|
162
|
-
const total = data["RESULT_LENGTH"];
|
|
163
|
-
if ((options == null ? void 0 : options.cursor) && (options == null ? void 0 : options.limit)) {
|
|
164
|
-
const prevCursor = options.cursor === "0" ? null : (parseInt(options.cursor) - 1).toString();
|
|
165
|
-
const nextCursor = total <= (parseInt(options.cursor) + 1) * options.limit ? null : (parseInt(options.cursor) + 1).toString();
|
|
166
|
-
return {
|
|
167
|
-
results,
|
|
168
|
-
total,
|
|
169
|
-
prevCursor,
|
|
170
|
-
nextCursor
|
|
171
|
-
};
|
|
172
|
-
} else if (!(options == null ? void 0 : options.cursor) && (options == null ? void 0 : options.limit)) {
|
|
173
|
-
const prevCursor = null;
|
|
174
|
-
const nextCursor = total <= options.limit ? null : "1";
|
|
175
|
-
return {
|
|
176
|
-
results,
|
|
177
|
-
total,
|
|
178
|
-
prevCursor,
|
|
179
|
-
nextCursor
|
|
180
|
-
};
|
|
181
|
-
} else {
|
|
182
|
-
return {
|
|
183
|
-
results,
|
|
184
|
-
total,
|
|
185
|
-
prevCursor: null,
|
|
186
|
-
nextCursor: null
|
|
187
|
-
};
|
|
188
|
-
}
|
|
189
|
-
};
|
|
190
|
-
export {
|
|
191
|
-
optionsToSearchIndexOptions,
|
|
192
|
-
parseSearchIndexResponse,
|
|
193
|
-
processDocumentForIndexing,
|
|
194
|
-
queryToSearchIndexQuery
|
|
195
|
-
};
|