jekyll-client-search 0.3.5 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +107 -1
- data/README.developer.md +30 -3
- data/README.md +209 -32
- data/assets/client-search-dropdown.js +57 -27
- data/assets/query-embedders/ollama-api.js +1 -1
- data/lib/jekyll/client_search/configuration.rb +11 -10
- data/lib/jekyll/client_search/configuration_accessors.rb +13 -0
- data/lib/jekyll/client_search/dropdown_tag.rb +7 -3
- data/lib/jekyll/client_search/embedding_configuration.rb +51 -15
- data/lib/jekyll/client_search/generator.rb +66 -19
- data/lib/jekyll/client_search/index_cache.rb +3 -2
- data/lib/jekyll/client_search/ollama_embedding_adapter.rb +47 -1
- data/lib/jekyll/client_search/query_embedder_configuration.rb +3 -0
- data/lib/jekyll/client_search/related_analyzer.rb +4 -1
- data/lib/jekyll/client_search/related_configuration.rb +15 -0
- data/lib/jekyll/client_search/semantic_configuration.rb +46 -0
- data/lib/jekyll/client_search/version.rb +1 -1
- data/lib/jekyll/client_search.rb +1 -0
- metadata +2 -1
|
@@ -9,8 +9,8 @@
|
|
|
9
9
|
* Features:
|
|
10
10
|
* - Lazy index loading (fetches search-index.json on first keystroke)
|
|
11
11
|
* - Two-stage search (AND first, fuzzy OR fallback) via engine adapter
|
|
12
|
-
* -
|
|
13
|
-
* -
|
|
12
|
+
* - ARIA combobox + listbox with plain-text options and optional icons
|
|
13
|
+
* - Active-descendant keyboard navigation (Arrow Up/Down, Enter, Escape)
|
|
14
14
|
* - Enter with no selection → redirect to redirect_url (?q=...)
|
|
15
15
|
* - Click outside or Escape → close dropdown
|
|
16
16
|
* - Multi-instance via [data-client-search-dropdown] attributes
|
|
@@ -20,6 +20,8 @@
|
|
|
20
20
|
var sharedIndex = null;
|
|
21
21
|
var sharedDocuments = null;
|
|
22
22
|
var indexLoadPromise = null;
|
|
23
|
+
var nextDropdownInstanceId = 0;
|
|
24
|
+
var initializedRoots = new WeakSet();
|
|
23
25
|
|
|
24
26
|
function safeUrl(value) {
|
|
25
27
|
if (value === "#") {
|
|
@@ -116,25 +118,27 @@
|
|
|
116
118
|
});
|
|
117
119
|
}
|
|
118
120
|
|
|
119
|
-
function createResultItem(match, documents, iconField) {
|
|
121
|
+
function createResultItem(match, documents, iconField, optionId) {
|
|
120
122
|
var entry = documents.get(match.ref);
|
|
121
123
|
if (!entry) {
|
|
122
124
|
return null;
|
|
123
125
|
}
|
|
124
126
|
|
|
125
|
-
var
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
127
|
+
var option = document.createElement("li");
|
|
128
|
+
option.id = optionId;
|
|
129
|
+
option.setAttribute("role", "option");
|
|
130
|
+
option.setAttribute("aria-selected", "false");
|
|
131
|
+
option.className = "client-search-dropdown-item";
|
|
132
|
+
option.dataset.url = safeUrl(entry.url);
|
|
129
133
|
|
|
130
134
|
if (entry.source) {
|
|
131
|
-
|
|
135
|
+
option.dataset.source = entry.source;
|
|
132
136
|
}
|
|
133
137
|
if (entry.categories && entry.categories.length) {
|
|
134
|
-
|
|
138
|
+
option.dataset.categories = entry.categories.join(" ");
|
|
135
139
|
}
|
|
136
140
|
if (entry.tags && entry.tags.length) {
|
|
137
|
-
|
|
141
|
+
option.dataset.tags = entry.tags.join(" ");
|
|
138
142
|
}
|
|
139
143
|
Object.keys(entry).forEach(function (key) {
|
|
140
144
|
var skip = ["id", "title", "url", "excerpt", "content",
|
|
@@ -145,7 +149,7 @@
|
|
|
145
149
|
}
|
|
146
150
|
var value = entry[key];
|
|
147
151
|
if (typeof value === "string" || typeof value === "number") {
|
|
148
|
-
|
|
152
|
+
option.dataset[toCamelCase(key)] = String(value);
|
|
149
153
|
}
|
|
150
154
|
});
|
|
151
155
|
|
|
@@ -153,22 +157,21 @@
|
|
|
153
157
|
var icon = document.createElement("img");
|
|
154
158
|
icon.className = "client-search-dropdown-icon";
|
|
155
159
|
icon.src = safeUrl(entry[iconField]);
|
|
156
|
-
icon.alt =
|
|
160
|
+
icon.alt = "";
|
|
161
|
+
icon.setAttribute("aria-hidden", "true");
|
|
157
162
|
icon.loading = "lazy";
|
|
158
163
|
icon.style.width = "1em";
|
|
159
164
|
icon.style.height = "1em";
|
|
160
165
|
icon.style.verticalAlign = "middle";
|
|
161
|
-
icon
|
|
162
|
-
li.appendChild(icon);
|
|
166
|
+
option.appendChild(icon);
|
|
163
167
|
}
|
|
164
168
|
|
|
165
|
-
var
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
li.appendChild(link);
|
|
169
|
+
var title = document.createElement("span");
|
|
170
|
+
title.className = "client-search-dropdown-link";
|
|
171
|
+
title.textContent = entry.title;
|
|
172
|
+
option.appendChild(title);
|
|
170
173
|
|
|
171
|
-
return
|
|
174
|
+
return option;
|
|
172
175
|
}
|
|
173
176
|
|
|
174
177
|
function DropdownInstance(root, config) {
|
|
@@ -176,6 +179,7 @@
|
|
|
176
179
|
this.form = root.querySelector("[data-cs-dropdown-form]");
|
|
177
180
|
this.input = root.querySelector("[data-cs-dropdown-input]");
|
|
178
181
|
this.results = root.querySelector("[data-cs-dropdown-results]");
|
|
182
|
+
this.listbox = this.results.closest('[role="listbox"]') || this.results;
|
|
179
183
|
this.config = config;
|
|
180
184
|
this.maxItems = Number.parseInt(this.results.dataset.maxItems, 10) || config.maxItems || 5;
|
|
181
185
|
this.selectedIndex = -1;
|
|
@@ -183,6 +187,9 @@
|
|
|
183
187
|
this.debounceTimer = null;
|
|
184
188
|
this.renderVersion = 0;
|
|
185
189
|
this.adapter = null;
|
|
190
|
+
this.optionIdPrefix = this.results.id
|
|
191
|
+
? this.results.id + "-option"
|
|
192
|
+
: "cs-dropdown-" + (++nextDropdownInstanceId) + "-option";
|
|
186
193
|
}
|
|
187
194
|
|
|
188
195
|
DropdownInstance.prototype.init = function () {
|
|
@@ -190,6 +197,11 @@
|
|
|
190
197
|
return;
|
|
191
198
|
}
|
|
192
199
|
|
|
200
|
+
this.results.setAttribute("aria-hidden", "true");
|
|
201
|
+
this.listbox.setAttribute("aria-hidden", "true");
|
|
202
|
+
this.input.setAttribute("aria-expanded", "false");
|
|
203
|
+
this.input.removeAttribute("aria-activedescendant");
|
|
204
|
+
|
|
193
205
|
var self = this;
|
|
194
206
|
|
|
195
207
|
this.input.addEventListener("input", function () {
|
|
@@ -212,9 +224,9 @@
|
|
|
212
224
|
});
|
|
213
225
|
|
|
214
226
|
this.results.addEventListener("click", function (event) {
|
|
215
|
-
var
|
|
216
|
-
if (
|
|
217
|
-
window.location.href =
|
|
227
|
+
var option = event.target.closest('[role="option"]');
|
|
228
|
+
if (option && option.dataset.url && option.dataset.url !== "#") {
|
|
229
|
+
window.location.href = option.dataset.url;
|
|
218
230
|
}
|
|
219
231
|
});
|
|
220
232
|
};
|
|
@@ -223,10 +235,10 @@
|
|
|
223
235
|
var self = this;
|
|
224
236
|
this.renderVersion += 1;
|
|
225
237
|
clearTimeout(this.debounceTimer);
|
|
238
|
+
this.hide();
|
|
226
239
|
|
|
227
240
|
var query = this.input.value.trim();
|
|
228
241
|
if (!query || query.length < this.config.minChars) {
|
|
229
|
-
this.hide();
|
|
230
242
|
return;
|
|
231
243
|
}
|
|
232
244
|
|
|
@@ -264,7 +276,6 @@
|
|
|
264
276
|
break;
|
|
265
277
|
case "Escape":
|
|
266
278
|
this.hide();
|
|
267
|
-
this.input.blur();
|
|
268
279
|
break;
|
|
269
280
|
case "Tab":
|
|
270
281
|
this.hide();
|
|
@@ -318,6 +329,7 @@
|
|
|
318
329
|
this.results.replaceChildren();
|
|
319
330
|
this.currentItems = [];
|
|
320
331
|
this.selectedIndex = -1;
|
|
332
|
+
this.input.removeAttribute("aria-activedescendant");
|
|
321
333
|
|
|
322
334
|
if (!matches || matches.length === 0) {
|
|
323
335
|
this.hide();
|
|
@@ -328,7 +340,12 @@
|
|
|
328
340
|
var count = Math.min(matches.length, this.maxItems);
|
|
329
341
|
|
|
330
342
|
for (var i = 0; i < count; i++) {
|
|
331
|
-
var item = createResultItem(
|
|
343
|
+
var item = createResultItem(
|
|
344
|
+
matches[i],
|
|
345
|
+
documents,
|
|
346
|
+
iconField,
|
|
347
|
+
this.optionIdPrefix + "-" + i
|
|
348
|
+
);
|
|
332
349
|
if (item) {
|
|
333
350
|
this.results.appendChild(item);
|
|
334
351
|
this.currentItems.push(item);
|
|
@@ -353,18 +370,26 @@
|
|
|
353
370
|
}
|
|
354
371
|
}
|
|
355
372
|
if (this.currentItems[index]) {
|
|
356
|
-
this.currentItems[index].
|
|
373
|
+
this.input.setAttribute("aria-activedescendant", this.currentItems[index].id);
|
|
374
|
+
if (typeof this.currentItems[index].scrollIntoView === "function") {
|
|
375
|
+
this.currentItems[index].scrollIntoView({ block: "nearest" });
|
|
376
|
+
}
|
|
377
|
+
} else {
|
|
378
|
+
this.input.removeAttribute("aria-activedescendant");
|
|
357
379
|
}
|
|
358
380
|
};
|
|
359
381
|
|
|
360
382
|
DropdownInstance.prototype.show = function () {
|
|
361
383
|
this.results.setAttribute("aria-hidden", "false");
|
|
384
|
+
this.listbox.setAttribute("aria-hidden", "false");
|
|
362
385
|
this.input.setAttribute("aria-expanded", "true");
|
|
363
386
|
};
|
|
364
387
|
|
|
365
388
|
DropdownInstance.prototype.hide = function () {
|
|
366
389
|
this.results.setAttribute("aria-hidden", "true");
|
|
390
|
+
this.listbox.setAttribute("aria-hidden", "true");
|
|
367
391
|
this.input.setAttribute("aria-expanded", "false");
|
|
392
|
+
this.input.removeAttribute("aria-activedescendant");
|
|
368
393
|
this.results.replaceChildren();
|
|
369
394
|
this.currentItems = [];
|
|
370
395
|
this.selectedIndex = -1;
|
|
@@ -389,6 +414,11 @@
|
|
|
389
414
|
|
|
390
415
|
var roots = document.querySelectorAll("[data-client-search-dropdown]");
|
|
391
416
|
roots.forEach(function (root) {
|
|
417
|
+
if (initializedRoots.has(root)) {
|
|
418
|
+
return;
|
|
419
|
+
}
|
|
420
|
+
initializedRoots.add(root);
|
|
421
|
+
|
|
392
422
|
var instance = new DropdownInstance(root, mergedConfig);
|
|
393
423
|
instance.init();
|
|
394
424
|
});
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
"use strict";
|
|
3
3
|
|
|
4
4
|
var config = window.ClientSearchEmbedderConfig || {};
|
|
5
|
-
var model = config.model || config.buildModel || "embeddinggemma:
|
|
5
|
+
var model = config.model || config.buildModel || "embeddinggemma-2:570m";
|
|
6
6
|
var apiUrl = config.apiUrl || "http://localhost:11434/api/embed";
|
|
7
7
|
var activeController = null;
|
|
8
8
|
|
|
@@ -27,7 +27,7 @@ module Jekyll
|
|
|
27
27
|
"icon_field" => "icon_url",
|
|
28
28
|
"embedding" => {
|
|
29
29
|
"enabled" => false,
|
|
30
|
-
"model" => "embeddinggemma:
|
|
30
|
+
"model" => "embeddinggemma-2:570m",
|
|
31
31
|
"base_url" => "http://localhost:11434",
|
|
32
32
|
"connect_timeout" => 5,
|
|
33
33
|
"read_timeout" => 120,
|
|
@@ -41,9 +41,7 @@ module Jekyll
|
|
|
41
41
|
def initialize(site)
|
|
42
42
|
configured = normalize_configured(site.config["client_search"])
|
|
43
43
|
@values = DEFAULTS.merge(configured)
|
|
44
|
-
|
|
45
|
-
@related = RelatedConfiguration.new(configured["related"] || {})
|
|
46
|
-
@dropdown = DropdownConfiguration.new(configured["dropdown"] || {})
|
|
44
|
+
build_sub_configurations(configured)
|
|
47
45
|
merge_embedding_config(configured["embedding"])
|
|
48
46
|
validate_engine!
|
|
49
47
|
return unless embedding_enabled?
|
|
@@ -54,6 +52,13 @@ module Jekyll
|
|
|
54
52
|
|
|
55
53
|
private
|
|
56
54
|
|
|
55
|
+
def build_sub_configurations(configured)
|
|
56
|
+
@live_search = LiveSearchConfiguration.new(configured["live_search"] || {}, engine: engine)
|
|
57
|
+
@related = RelatedConfiguration.new(configured["related"] || {})
|
|
58
|
+
@semantic = SemanticConfiguration.new(configured["semantic"] || {})
|
|
59
|
+
@dropdown = DropdownConfiguration.new(configured["dropdown"] || {})
|
|
60
|
+
end
|
|
61
|
+
|
|
57
62
|
def normalize_configured(value)
|
|
58
63
|
return { "enabled" => false } if value == false
|
|
59
64
|
return value if value.is_a?(Hash)
|
|
@@ -70,13 +75,9 @@ module Jekyll
|
|
|
70
75
|
"client_search embedding configuration must be a mapping"
|
|
71
76
|
end
|
|
72
77
|
@values["embedding"] = DEFAULTS.fetch("embedding").merge(embedding)
|
|
73
|
-
merge_query_embedder_config(embedding["query_embedder"])
|
|
74
|
-
end
|
|
75
|
-
|
|
76
|
-
def merge_query_embedder_config(configured_query_embedder)
|
|
77
78
|
@query_embedder = QueryEmbedderConfiguration.new(
|
|
78
|
-
|
|
79
|
-
build_model:
|
|
79
|
+
embedding["query_embedder"] || {},
|
|
80
|
+
build_model: embedding_index_model,
|
|
80
81
|
build_base_url: embedding_base_url,
|
|
81
82
|
query_prefix: embedding_query_prefix
|
|
82
83
|
)
|
|
@@ -20,6 +20,19 @@ module Jekyll
|
|
|
20
20
|
@related
|
|
21
21
|
end
|
|
22
22
|
|
|
23
|
+
# The Ollama model whose vectors are written into the search index and
|
|
24
|
+
# must match the browser query embedder's vector space. Defaults to
|
|
25
|
+
# embedding.model; override with semantic.index_model.
|
|
26
|
+
def embedding_index_model
|
|
27
|
+
@semantic.index_model || embedding_model
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
# The Ollama model used for related-article similarity analysis.
|
|
31
|
+
# Defaults to embedding.model; override with related.index_model.
|
|
32
|
+
def embedding_related_model
|
|
33
|
+
@related.index_model || embedding_model
|
|
34
|
+
end
|
|
35
|
+
|
|
23
36
|
def query_embedder_type
|
|
24
37
|
@query_embedder.type
|
|
25
38
|
end
|
|
@@ -87,13 +87,17 @@ module Jekyll
|
|
|
87
87
|
<div class="client-search-dropdown" data-client-search-dropdown>
|
|
88
88
|
<form role="search" autocomplete="off" data-cs-dropdown-form>
|
|
89
89
|
<label class="sr-only" for="#{@input_id}">Search</label>
|
|
90
|
-
<input id="#{@input_id}" type="search" name="q"
|
|
90
|
+
<input id="#{@input_id}" type="search" name="q" role="combobox"
|
|
91
|
+
aria-autocomplete="list" aria-haspopup="listbox"
|
|
91
92
|
placeholder="Search…" autocomplete="off"
|
|
92
93
|
aria-expanded="false" aria-controls="#{@results_id}"
|
|
93
94
|
data-cs-dropdown-input>
|
|
94
95
|
</form>
|
|
95
|
-
<
|
|
96
|
-
|
|
96
|
+
<div id="#{@results_id}" role="listbox" aria-label="Search suggestions"
|
|
97
|
+
aria-hidden="true">
|
|
98
|
+
<ul role="presentation" aria-hidden="true"
|
|
99
|
+
data-cs-dropdown-results data-max-items="#{max}"></ul>
|
|
100
|
+
</div>
|
|
97
101
|
</div>
|
|
98
102
|
HTML
|
|
99
103
|
end
|
|
@@ -9,9 +9,25 @@ module Jekyll
|
|
|
9
9
|
"document" => "title: none | text: ",
|
|
10
10
|
"query" => "task: search result | query: "
|
|
11
11
|
},
|
|
12
|
+
"embeddinggemma-2" => {
|
|
13
|
+
"document" => "title: none | text: ",
|
|
14
|
+
"query" => "task: search result | query: "
|
|
15
|
+
},
|
|
12
16
|
"nomic-embed-text" => {
|
|
13
17
|
"document" => "search_document: ",
|
|
14
18
|
"query" => "search_query: "
|
|
19
|
+
},
|
|
20
|
+
"voyage-4-nano" => {
|
|
21
|
+
"document" => "Represent the document for retrieval: ",
|
|
22
|
+
"query" => "Represent the query for retrieving supporting documents: "
|
|
23
|
+
},
|
|
24
|
+
"qwen3-embedding" => {
|
|
25
|
+
"document" => "",
|
|
26
|
+
"query" => "Instruct: Given a web search query, retrieve relevant passages that answer the query\nQuery: "
|
|
27
|
+
},
|
|
28
|
+
"bge-large" => {
|
|
29
|
+
"document" => "",
|
|
30
|
+
"query" => "Represent this sentence for searching relevant passages: "
|
|
15
31
|
}
|
|
16
32
|
}.freeze
|
|
17
33
|
|
|
@@ -44,43 +60,63 @@ module Jekyll
|
|
|
44
60
|
configured.nil? ? engine == "semantic" : configured == true
|
|
45
61
|
end
|
|
46
62
|
|
|
63
|
+
# Document prefix for the index model (semantic.index_model).
|
|
47
64
|
def embedding_document_prefix
|
|
48
|
-
configured_prefix("
|
|
65
|
+
configured_prefix("document", model: embedding_index_model, override: @semantic.document_prefix)
|
|
49
66
|
end
|
|
50
67
|
|
|
68
|
+
# Query prefix for the index model — must match the browser query
|
|
69
|
+
# embedder's vector space.
|
|
51
70
|
def embedding_query_prefix
|
|
52
|
-
configured_prefix("
|
|
71
|
+
configured_prefix("query", model: embedding_index_model, override: @semantic.query_prefix)
|
|
72
|
+
end
|
|
73
|
+
|
|
74
|
+
# Document prefix for the related-analysis model (related.index_model).
|
|
75
|
+
def embedding_related_document_prefix
|
|
76
|
+
configured_prefix("document", model: embedding_related_model)
|
|
53
77
|
end
|
|
54
78
|
|
|
55
79
|
def embedding_identity
|
|
80
|
+
embedding_identity_for(model: embedding_index_model,
|
|
81
|
+
document_prefix: embedding_document_prefix,
|
|
82
|
+
query_prefix: embedding_query_prefix)
|
|
83
|
+
end
|
|
84
|
+
|
|
85
|
+
def embedding_related_identity
|
|
86
|
+
embedding_identity_for(model: embedding_related_model,
|
|
87
|
+
document_prefix: embedding_related_document_prefix,
|
|
88
|
+
query_prefix: configured_prefix("query", model: embedding_related_model))
|
|
89
|
+
end
|
|
90
|
+
|
|
91
|
+
private
|
|
92
|
+
|
|
93
|
+
def embedding_identity_for(model:, document_prefix:, query_prefix:)
|
|
56
94
|
{
|
|
57
95
|
"provider" => "ollama",
|
|
58
|
-
"model" =>
|
|
96
|
+
"model" => model,
|
|
59
97
|
"base_url" => embedding_base_url,
|
|
60
|
-
"document_prefix" =>
|
|
61
|
-
"query_prefix" =>
|
|
98
|
+
"document_prefix" => document_prefix,
|
|
99
|
+
"query_prefix" => query_prefix,
|
|
62
100
|
"schema" => 2
|
|
63
101
|
}
|
|
64
102
|
end
|
|
65
103
|
|
|
66
|
-
|
|
104
|
+
def configured_prefix(role, model:, override: nil)
|
|
105
|
+
return override.to_s if override
|
|
67
106
|
|
|
68
|
-
def configured_prefix(config_key, role)
|
|
69
107
|
embedding = @values.fetch("embedding")
|
|
70
|
-
|
|
108
|
+
config_key = "#{role}_prefix"
|
|
109
|
+
return embedding[config_key].to_s if model == embedding_model && embedding.key?(config_key)
|
|
71
110
|
|
|
72
|
-
|
|
73
|
-
PREFIXES.fetch(model_name, {}).fetch(role, "")
|
|
111
|
+
PREFIXES.fetch(model.split(":").first, {}).fetch(role, "")
|
|
74
112
|
end
|
|
75
113
|
|
|
76
114
|
def positive_timeout(key)
|
|
77
115
|
value = @values.fetch("embedding").fetch(key).to_f
|
|
78
|
-
|
|
79
|
-
raise Jekyll::Errors::FatalException,
|
|
80
|
-
"embedding #{key} must be a finite value greater than zero"
|
|
81
|
-
end
|
|
116
|
+
return value if value.positive? && value.finite?
|
|
82
117
|
|
|
83
|
-
|
|
118
|
+
raise Jekyll::Errors::FatalException,
|
|
119
|
+
"embedding #{key} must be a finite value greater than zero"
|
|
84
120
|
end
|
|
85
121
|
|
|
86
122
|
def validate_embedding!
|
|
@@ -25,6 +25,7 @@ module Jekyll
|
|
|
25
25
|
|
|
26
26
|
def add_generated_pages(site, documents, configuration)
|
|
27
27
|
add_related_page(site, documents, configuration) if configuration.related_enabled?
|
|
28
|
+
documents.each { |document| document.delete("related_embedding") }
|
|
28
29
|
remove_embeddings(documents) unless configuration.embedding_include_in_index?
|
|
29
30
|
site.pages << SearchIndexPage.new(site, configuration.output, documents)
|
|
30
31
|
site.pages << RuntimeConfigPage.new(site, configuration) if configuration.copy_runtime?
|
|
@@ -39,18 +40,60 @@ module Jekyll
|
|
|
39
40
|
end
|
|
40
41
|
|
|
41
42
|
def enhance_with_embeddings(site, documents, configuration)
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
43
|
+
embedding_jobs(configuration).each do |job|
|
|
44
|
+
cache = IndexCache.new(site.source, embedding_identity: job[:identity],
|
|
45
|
+
file_name: job[:cache_file])
|
|
46
|
+
adapter = build_embedding_adapter(configuration, model: job[:model])
|
|
47
|
+
|
|
48
|
+
documents.each { |entry| embed_document(entry, cache, adapter, configuration, job) }
|
|
49
|
+
cache.prune(documents.map { |doc| doc["id"] })
|
|
50
|
+
cache.save
|
|
51
|
+
end
|
|
48
52
|
documents
|
|
49
53
|
end
|
|
50
54
|
|
|
51
|
-
|
|
55
|
+
# Each job embeds every document once per model into a target field.
|
|
56
|
+
# The index model writes "embedding" (kept when include_in_index); a
|
|
57
|
+
# divergent related model writes "related_embedding" (stripped before
|
|
58
|
+
# the index is emitted). Identical models share a single pass.
|
|
59
|
+
def embedding_jobs(configuration)
|
|
60
|
+
jobs = []
|
|
61
|
+
jobs << index_embedding_job(configuration) if configuration.embedding_include_in_index?
|
|
62
|
+
jobs.concat(related_embedding_jobs(configuration)) if configuration.related_enabled?
|
|
63
|
+
jobs
|
|
64
|
+
end
|
|
65
|
+
|
|
66
|
+
def index_embedding_job(configuration)
|
|
67
|
+
{
|
|
68
|
+
model: configuration.embedding_index_model,
|
|
69
|
+
field: "embedding",
|
|
70
|
+
document_prefix: configuration.embedding_document_prefix,
|
|
71
|
+
identity: configuration.embedding_identity,
|
|
72
|
+
cache_file: IndexCache::CACHE_FILE
|
|
73
|
+
}
|
|
74
|
+
end
|
|
75
|
+
|
|
76
|
+
def related_embedding_jobs(configuration)
|
|
77
|
+
return [related_embedding_job("embedding", IndexCache::CACHE_FILE, configuration)] unless
|
|
78
|
+
configuration.embedding_include_in_index?
|
|
79
|
+
return [] if configuration.embedding_related_model == configuration.embedding_index_model
|
|
80
|
+
|
|
81
|
+
[related_embedding_job("related_embedding", IndexCache::RELATED_CACHE_FILE, configuration)]
|
|
82
|
+
end
|
|
83
|
+
|
|
84
|
+
def related_embedding_job(field, cache_file, configuration)
|
|
85
|
+
{
|
|
86
|
+
model: configuration.embedding_related_model,
|
|
87
|
+
field: field,
|
|
88
|
+
document_prefix: configuration.embedding_related_document_prefix,
|
|
89
|
+
identity: configuration.embedding_related_identity,
|
|
90
|
+
cache_file: cache_file
|
|
91
|
+
}
|
|
92
|
+
end
|
|
93
|
+
|
|
94
|
+
def build_embedding_adapter(configuration, model: configuration.embedding_model)
|
|
52
95
|
OllamaEmbeddingAdapter.new(
|
|
53
|
-
model:
|
|
96
|
+
model: model,
|
|
54
97
|
base_url: configuration.embedding_base_url,
|
|
55
98
|
connect_timeout: configuration.embedding_connect_timeout,
|
|
56
99
|
read_timeout: configuration.embedding_read_timeout
|
|
@@ -66,17 +109,13 @@ module Jekyll
|
|
|
66
109
|
documents.each { |document| document.delete("embedding") }
|
|
67
110
|
end
|
|
68
111
|
|
|
69
|
-
def embed_document(document, cache, adapter, configuration)
|
|
70
|
-
hash = IndexCache.content_hash(document)
|
|
71
|
-
|
|
72
|
-
if cached && cached["embedding"]
|
|
73
|
-
document["embedding"] = cached["embedding"]
|
|
74
|
-
return
|
|
75
|
-
end
|
|
112
|
+
def embed_document(document, cache, adapter, configuration, job)
|
|
113
|
+
hash = IndexCache.content_hash(document.except("embedding", "related_embedding"))
|
|
114
|
+
return if embedding_restored?(document, cache, hash, job[:field])
|
|
76
115
|
|
|
77
|
-
embedding = adapter.embed(embedding_text(document,
|
|
116
|
+
embedding = adapter.embed(embedding_text(document, job[:document_prefix]))
|
|
78
117
|
if embedding
|
|
79
|
-
document[
|
|
118
|
+
document[job[:field]] = embedding
|
|
80
119
|
cache.store(document["id"], hash, embedding)
|
|
81
120
|
elsif configuration.embedding_fail_on_error?
|
|
82
121
|
raise Jekyll::Errors::FatalException,
|
|
@@ -84,9 +123,17 @@ module Jekyll
|
|
|
84
123
|
end
|
|
85
124
|
end
|
|
86
125
|
|
|
87
|
-
def
|
|
126
|
+
def embedding_restored?(document, cache, hash, field)
|
|
127
|
+
cached = cache.lookup(document["id"], hash)
|
|
128
|
+
return false unless cached && cached["embedding"]
|
|
129
|
+
|
|
130
|
+
document[field] = cached["embedding"]
|
|
131
|
+
true
|
|
132
|
+
end
|
|
133
|
+
|
|
134
|
+
def embedding_text(document, document_prefix)
|
|
88
135
|
text = [document["title"], document["excerpt"], document["content"]].compact.join(" ")
|
|
89
|
-
"#{
|
|
136
|
+
"#{document_prefix}#{text}"
|
|
90
137
|
end
|
|
91
138
|
|
|
92
139
|
def collection_documents(site, configuration, builder)
|
|
@@ -12,11 +12,12 @@ module Jekyll
|
|
|
12
12
|
# should be git-ignored.
|
|
13
13
|
class IndexCache
|
|
14
14
|
CACHE_FILE = ".jekyll-client-search-cache.json"
|
|
15
|
+
RELATED_CACHE_FILE = ".jekyll-client-search-related-cache.json"
|
|
15
16
|
|
|
16
17
|
attr_reader :path
|
|
17
18
|
|
|
18
|
-
def initialize(site_source, embedding_identity: nil)
|
|
19
|
-
@path = File.join(site_source,
|
|
19
|
+
def initialize(site_source, embedding_identity: nil, file_name: CACHE_FILE)
|
|
20
|
+
@path = File.join(site_source, file_name)
|
|
20
21
|
@embedding_identity = embedding_identity
|
|
21
22
|
@entries = load
|
|
22
23
|
end
|
|
@@ -11,6 +11,11 @@ module Jekyll
|
|
|
11
11
|
# enabled: true
|
|
12
12
|
# model: all-minilm # or nomic-embed-text, bge-m3, etc.
|
|
13
13
|
# base_url: http://localhost:11434
|
|
14
|
+
#
|
|
15
|
+
# A +localhost+ base_url is resolved once to the loopback literal
|
|
16
|
+
# (127.0.0.1 or ::1) that accepts a TCP connection, because Excon
|
|
17
|
+
# does not fall back across address families when the DNS name maps
|
|
18
|
+
# to a family the server is not listening on.
|
|
14
19
|
class OllamaEmbeddingAdapter
|
|
15
20
|
attr_reader :model, :base_url, :connect_timeout, :read_timeout
|
|
16
21
|
|
|
@@ -48,12 +53,53 @@ module Jekyll
|
|
|
48
53
|
@client ||= begin
|
|
49
54
|
require "ollama"
|
|
50
55
|
Ollama::Client.new(
|
|
51
|
-
base_url:
|
|
56
|
+
base_url: resolved_base_url,
|
|
52
57
|
connect_timeout: @connect_timeout,
|
|
53
58
|
read_timeout: @read_timeout
|
|
54
59
|
)
|
|
55
60
|
end
|
|
56
61
|
end
|
|
62
|
+
|
|
63
|
+
# "localhost" may resolve to ::1 while the server only listens on
|
|
64
|
+
# 127.0.0.1, or the reverse. Excon does not fall back across
|
|
65
|
+
# address families, so probe both loopback literals once and use
|
|
66
|
+
# the one that accepts a TCP connection. Any other host name is
|
|
67
|
+
# used exactly as configured.
|
|
68
|
+
def resolved_base_url
|
|
69
|
+
@resolved_base_url ||= resolve_localhost(@base_url)
|
|
70
|
+
end
|
|
71
|
+
|
|
72
|
+
def resolve_localhost(base_url)
|
|
73
|
+
require "uri"
|
|
74
|
+
uri = URI.parse(base_url)
|
|
75
|
+
return base_url unless uri.host.to_s.chomp(".").casecmp("localhost").zero?
|
|
76
|
+
|
|
77
|
+
resolved = loopback_base_url(uri)
|
|
78
|
+
return base_url unless resolved
|
|
79
|
+
|
|
80
|
+
Jekyll.logger.info "ClientSearch:", "resolved localhost to #{resolved}"
|
|
81
|
+
resolved
|
|
82
|
+
rescue URI::InvalidURIError
|
|
83
|
+
base_url
|
|
84
|
+
end
|
|
85
|
+
|
|
86
|
+
def loopback_base_url(uri)
|
|
87
|
+
%w[127.0.0.1 ::1].each do |address|
|
|
88
|
+
next unless tcp_reachable?(address, uri.port)
|
|
89
|
+
|
|
90
|
+
resolved = uri.dup
|
|
91
|
+
resolved.host = address.include?(":") ? "[#{address}]" : address
|
|
92
|
+
return resolved.to_s
|
|
93
|
+
end
|
|
94
|
+
nil
|
|
95
|
+
end
|
|
96
|
+
|
|
97
|
+
def tcp_reachable?(address, port)
|
|
98
|
+
require "socket"
|
|
99
|
+
Socket.tcp(address, port, connect_timeout: @connect_timeout) { true }
|
|
100
|
+
rescue SystemCallError, SocketError
|
|
101
|
+
false
|
|
102
|
+
end
|
|
57
103
|
end
|
|
58
104
|
end
|
|
59
105
|
end
|
|
@@ -8,6 +8,9 @@ module Jekyll
|
|
|
8
8
|
|
|
9
9
|
MODEL_MAP = {
|
|
10
10
|
"embeddinggemma:300m" => "onnx-community/embeddinggemma-300m-ONNX",
|
|
11
|
+
"embeddinggemma-2" => "onnx-community/embeddinggemma-2-ONNX",
|
|
12
|
+
"embeddinggemma-2:270m" => "onnx-community/embeddinggemma-2-ONNX",
|
|
13
|
+
"embeddinggemma-2:570m" => "onnx-community/embeddinggemma-2-ONNX",
|
|
11
14
|
"all-minilm" => "Xenova/all-MiniLM-L6-v2"
|
|
12
15
|
}.freeze
|
|
13
16
|
|
|
@@ -29,7 +29,10 @@ module Jekyll
|
|
|
29
29
|
|
|
30
30
|
def build_relation(source, target)
|
|
31
31
|
metadata = metadata_relation(source, target)
|
|
32
|
-
semantic_similarity = similarity(
|
|
32
|
+
semantic_similarity = similarity(
|
|
33
|
+
source["related_embedding"] || source["embedding"],
|
|
34
|
+
target["related_embedding"] || target["embedding"]
|
|
35
|
+
)
|
|
33
36
|
return unless metadata || semantic_match?(semantic_similarity)
|
|
34
37
|
|
|
35
38
|
relation = base_relation(target, metadata, semantic_similarity)
|