@gmickel/gno 1.8.0 → 1.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +6 -2
- package/assets/skill/SKILL.md +13 -0
- package/assets/skill/cli-reference.md +9 -0
- package/assets/skill/examples.md +21 -0
- package/assets/skill/mcp-reference.md +6 -0
- package/package.json +1 -1
- package/src/cli/commands/capture.ts +9 -6
- package/src/cli/commands/index-cmd.ts +17 -6
- package/src/cli/commands/shared.ts +17 -1
- package/src/cli/commands/update.ts +17 -6
- package/src/config/content-types.ts +140 -0
- package/src/config/defaults.ts +1 -0
- package/src/config/index.ts +14 -0
- package/src/config/loader.ts +11 -2
- package/src/config/types.ts +37 -1
- package/src/core/config-mutation.ts +14 -2
- package/src/core/note-presets.ts +61 -5
- package/src/ingestion/frontmatter.ts +77 -2
- package/src/ingestion/index.ts +2 -0
- package/src/ingestion/sync-options.ts +29 -0
- package/src/ingestion/sync.ts +104 -16
- package/src/ingestion/types.ts +14 -0
- package/src/mcp/tools/add-collection.ts +8 -5
- package/src/mcp/tools/capture.ts +5 -2
- package/src/mcp/tools/index-cmd.ts +13 -7
- package/src/mcp/tools/index.ts +8 -9
- package/src/mcp/tools/sync.ts +12 -6
- package/src/mcp/tools/workspace-write.ts +16 -10
- package/src/pipeline/hybrid.ts +2 -0
- package/src/pipeline/search.ts +2 -0
- package/src/pipeline/types.ts +2 -0
- package/src/pipeline/vsearch.ts +4 -0
- package/src/sdk/client.ts +51 -24
- package/src/serve/background-runtime.ts +18 -4
- package/src/serve/config-sync.ts +9 -2
- package/src/serve/routes/api.ts +50 -24
- package/src/serve/watch-service.ts +12 -2
- package/src/store/migrations/009-content-type-rule-fingerprint.ts +36 -0
- package/src/store/migrations/index.ts +12 -1
- package/src/store/sqlite/adapter.ts +29 -14
- package/src/store/types.ts +6 -0
package/src/pipeline/hybrid.ts
CHANGED
|
@@ -841,6 +841,8 @@ export async function searchHybrid(
|
|
|
841
841
|
score: candidate.blendedScore,
|
|
842
842
|
uri: doc.uri,
|
|
843
843
|
title: doc.title ?? undefined,
|
|
844
|
+
contentType: doc.contentType ?? undefined,
|
|
845
|
+
categories: doc.categories ?? undefined,
|
|
844
846
|
line: snippetChunk.startLine,
|
|
845
847
|
snippet,
|
|
846
848
|
snippetLanguage: chunk.language ?? undefined,
|
package/src/pipeline/search.ts
CHANGED
|
@@ -117,6 +117,8 @@ function buildSearchResult(ctx: BuildResultContext): SearchResult {
|
|
|
117
117
|
score: fts.score, // Raw score, normalized later as batch
|
|
118
118
|
uri: fts.uri ?? "",
|
|
119
119
|
title: fts.title,
|
|
120
|
+
contentType: fts.contentType,
|
|
121
|
+
categories: fts.categories,
|
|
120
122
|
line: chunk?.startLine,
|
|
121
123
|
snippet,
|
|
122
124
|
snippetLanguage: chunk?.language ?? undefined,
|
package/src/pipeline/types.ts
CHANGED
package/src/pipeline/vsearch.ts
CHANGED
|
@@ -227,6 +227,8 @@ export async function searchVectorWithEmbedding(
|
|
|
227
227
|
score,
|
|
228
228
|
uri: doc.uri,
|
|
229
229
|
title: doc.title ?? undefined,
|
|
230
|
+
contentType: doc.contentType ?? undefined,
|
|
231
|
+
categories: doc.categories ?? undefined,
|
|
230
232
|
line: chunk.startLine,
|
|
231
233
|
snippet: chunk.text,
|
|
232
234
|
snippetLanguage: chunk.language ?? undefined,
|
|
@@ -281,6 +283,8 @@ export async function searchVectorWithEmbedding(
|
|
|
281
283
|
score,
|
|
282
284
|
uri: doc.uri,
|
|
283
285
|
title: doc.title ?? undefined,
|
|
286
|
+
contentType: doc.contentType ?? undefined,
|
|
287
|
+
categories: doc.categories ?? undefined,
|
|
284
288
|
line: chunk.startLine,
|
|
285
289
|
snippet: fullContent ?? chunk.text,
|
|
286
290
|
snippetLanguage: chunk.language ?? undefined,
|
package/src/sdk/client.ts
CHANGED
|
@@ -44,7 +44,11 @@ import {
|
|
|
44
44
|
getIndexDbPath,
|
|
45
45
|
parseUri,
|
|
46
46
|
} from "../app/constants";
|
|
47
|
-
import {
|
|
47
|
+
import {
|
|
48
|
+
ConfigSchema,
|
|
49
|
+
loadConfig,
|
|
50
|
+
normalizeConfigContentTypes,
|
|
51
|
+
} from "../config";
|
|
48
52
|
import {
|
|
49
53
|
buildCaptureReceipt,
|
|
50
54
|
type CapturePlan,
|
|
@@ -70,7 +74,11 @@ import { resolveNotePreset } from "../core/note-presets";
|
|
|
70
74
|
import { extractSections } from "../core/sections";
|
|
71
75
|
import { normalizeStructuredQueryInput } from "../core/structured-query";
|
|
72
76
|
import { parseAndValidateTagFilter } from "../core/tags";
|
|
73
|
-
import {
|
|
77
|
+
import {
|
|
78
|
+
defaultSyncService,
|
|
79
|
+
type SyncResult,
|
|
80
|
+
withContentTypeRules,
|
|
81
|
+
} from "../ingestion";
|
|
74
82
|
import { updateFrontmatterTags } from "../ingestion/frontmatter";
|
|
75
83
|
import { LlmAdapter } from "../llm/nodeLlamaCpp/adapter";
|
|
76
84
|
import { resolveDownloadPolicy } from "../llm/policy";
|
|
@@ -141,7 +149,7 @@ async function resolveClientState(
|
|
|
141
149
|
parsed.error.issues[0]?.message ?? "Invalid config"
|
|
142
150
|
);
|
|
143
151
|
}
|
|
144
|
-
config = parsed.data;
|
|
152
|
+
config = normalizeConfigContentTypes(parsed.data).config;
|
|
145
153
|
configPath = null;
|
|
146
154
|
configSource = "inline";
|
|
147
155
|
} else {
|
|
@@ -695,10 +703,17 @@ class GnoClientImpl implements GnoClient {
|
|
|
695
703
|
async update(options: GnoUpdateOptions = {}): Promise<SyncResult> {
|
|
696
704
|
this.assertOpen();
|
|
697
705
|
const collections = this.getCollections(options.collection);
|
|
698
|
-
return defaultSyncService.syncAll(
|
|
699
|
-
|
|
700
|
-
|
|
701
|
-
|
|
706
|
+
return defaultSyncService.syncAll(
|
|
707
|
+
collections,
|
|
708
|
+
this.store,
|
|
709
|
+
withContentTypeRules(
|
|
710
|
+
{
|
|
711
|
+
gitPull: options.gitPull,
|
|
712
|
+
runUpdateCmd: true,
|
|
713
|
+
},
|
|
714
|
+
this.config
|
|
715
|
+
)
|
|
716
|
+
);
|
|
702
717
|
}
|
|
703
718
|
|
|
704
719
|
async embed(options: GnoEmbedOptions = {}): Promise<GnoEmbedResult> {
|
|
@@ -808,10 +823,13 @@ class GnoClientImpl implements GnoClient {
|
|
|
808
823
|
collection,
|
|
809
824
|
this.store,
|
|
810
825
|
[plan.relPath],
|
|
811
|
-
|
|
812
|
-
|
|
813
|
-
|
|
814
|
-
|
|
826
|
+
withContentTypeRules(
|
|
827
|
+
{
|
|
828
|
+
runUpdateCmd: false,
|
|
829
|
+
gitPull: false,
|
|
830
|
+
},
|
|
831
|
+
this.config
|
|
832
|
+
)
|
|
815
833
|
);
|
|
816
834
|
const syncResult = syncResults[0];
|
|
817
835
|
if (!syncResult || syncResult.status === "error") {
|
|
@@ -903,10 +921,13 @@ class GnoClientImpl implements GnoClient {
|
|
|
903
921
|
collection,
|
|
904
922
|
this.store,
|
|
905
923
|
[plan.relPath],
|
|
906
|
-
|
|
907
|
-
|
|
908
|
-
|
|
909
|
-
|
|
924
|
+
withContentTypeRules(
|
|
925
|
+
{
|
|
926
|
+
runUpdateCmd: false,
|
|
927
|
+
gitPull: false,
|
|
928
|
+
},
|
|
929
|
+
this.config
|
|
930
|
+
)
|
|
910
931
|
);
|
|
911
932
|
const syncResult = syncResults[0];
|
|
912
933
|
const docResult = await this.store.getDocument(
|
|
@@ -987,9 +1008,11 @@ class GnoClientImpl implements GnoClient {
|
|
|
987
1008
|
const currentPath = `${collection.path}/${storedDoc.relPath}`;
|
|
988
1009
|
const nextPath = `${collection.path}/${plan.nextRelPath}`;
|
|
989
1010
|
await renameFilePath(currentPath, nextPath);
|
|
990
|
-
await defaultSyncService.syncCollection(
|
|
991
|
-
|
|
992
|
-
|
|
1011
|
+
await defaultSyncService.syncCollection(
|
|
1012
|
+
collection,
|
|
1013
|
+
this.store,
|
|
1014
|
+
withContentTypeRules({ runUpdateCmd: false }, this.config)
|
|
1015
|
+
);
|
|
993
1016
|
const linksResult = await this.store.getLinksForDoc(storedDoc.id);
|
|
994
1017
|
const backlinksResult = await this.store.getBacklinksForDoc(storedDoc.id);
|
|
995
1018
|
if (!linksResult.ok || !backlinksResult.ok) {
|
|
@@ -1044,9 +1067,11 @@ class GnoClientImpl implements GnoClient {
|
|
|
1044
1067
|
const nextPath = `${collection.path}/${plan.nextRelPath}`;
|
|
1045
1068
|
await mkdir(dirname(nextPath), { recursive: true });
|
|
1046
1069
|
await renameFilePath(currentPath, nextPath);
|
|
1047
|
-
await defaultSyncService.syncCollection(
|
|
1048
|
-
|
|
1049
|
-
|
|
1070
|
+
await defaultSyncService.syncCollection(
|
|
1071
|
+
collection,
|
|
1072
|
+
this.store,
|
|
1073
|
+
withContentTypeRules({ runUpdateCmd: false }, this.config)
|
|
1074
|
+
);
|
|
1050
1075
|
const linksResult = await this.store.getLinksForDoc(storedDoc.id);
|
|
1051
1076
|
const backlinksResult = await this.store.getBacklinksForDoc(storedDoc.id);
|
|
1052
1077
|
if (!linksResult.ok || !backlinksResult.ok) {
|
|
@@ -1113,9 +1138,11 @@ class GnoClientImpl implements GnoClient {
|
|
|
1113
1138
|
const nextPath = `${collection.path}/${plan.nextRelPath}`;
|
|
1114
1139
|
await mkdir(dirname(nextPath), { recursive: true });
|
|
1115
1140
|
await copyFilePath(currentPath, nextPath);
|
|
1116
|
-
await defaultSyncService.syncCollection(
|
|
1117
|
-
|
|
1118
|
-
|
|
1141
|
+
await defaultSyncService.syncCollection(
|
|
1142
|
+
collection,
|
|
1143
|
+
this.store,
|
|
1144
|
+
withContentTypeRules({ runUpdateCmd: false }, this.config)
|
|
1145
|
+
);
|
|
1119
1146
|
const linksResult = await this.store.getLinksForDoc(storedDoc.id);
|
|
1120
1147
|
const backlinksResult = await this.store.getBacklinksForDoc(storedDoc.id);
|
|
1121
1148
|
if (!linksResult.ok || !backlinksResult.ok) {
|
|
@@ -11,11 +11,13 @@ import type {
|
|
|
11
11
|
import { getIndexDbPath } from "../app/constants";
|
|
12
12
|
import {
|
|
13
13
|
ensureDirectories,
|
|
14
|
+
formatConfigWarnings,
|
|
14
15
|
getConfigPaths,
|
|
15
16
|
isInitialized,
|
|
16
17
|
loadConfig,
|
|
17
18
|
} from "../config";
|
|
18
19
|
import { defaultSyncService } from "../ingestion";
|
|
20
|
+
import { withContentTypeRules } from "../ingestion";
|
|
19
21
|
import { getActivePreset } from "../llm/registry";
|
|
20
22
|
import { SqliteAdapter } from "../store/sqlite/adapter";
|
|
21
23
|
import {
|
|
@@ -79,6 +81,7 @@ type BackgroundRuntimeDeps = {
|
|
|
79
81
|
scheduler: EmbedScheduler | null;
|
|
80
82
|
eventBus?: DocumentEventBus | null;
|
|
81
83
|
callbacks?: CollectionWatchCallbacks;
|
|
84
|
+
syncOptions?: Parameters<typeof withContentTypeRules>[0];
|
|
82
85
|
}) => CollectionWatchService;
|
|
83
86
|
};
|
|
84
87
|
|
|
@@ -103,6 +106,9 @@ export async function startBackgroundRuntime(
|
|
|
103
106
|
if (!configResult.ok) {
|
|
104
107
|
return { success: false, error: configResult.error.message };
|
|
105
108
|
}
|
|
109
|
+
for (const warning of formatConfigWarnings(configResult.warnings)) {
|
|
110
|
+
console.warn(warning);
|
|
111
|
+
}
|
|
106
112
|
const config = configResult.value;
|
|
107
113
|
|
|
108
114
|
if (options.requireCollections && config.collections.length === 0) {
|
|
@@ -171,6 +177,7 @@ export async function startBackgroundRuntime(
|
|
|
171
177
|
scheduler,
|
|
172
178
|
eventBus: options.eventBus ?? null,
|
|
173
179
|
callbacks: options.watchCallbacks,
|
|
180
|
+
syncOptions: withContentTypeRules({}, config),
|
|
174
181
|
});
|
|
175
182
|
watchService.start();
|
|
176
183
|
ctxHolder.watchService = watchService;
|
|
@@ -189,10 +196,17 @@ export async function startBackgroundRuntime(
|
|
|
189
196
|
eventBus: options.eventBus ?? null,
|
|
190
197
|
watchService,
|
|
191
198
|
async syncAll(syncOptions = {}) {
|
|
192
|
-
const syncResult = await syncAllService(
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
199
|
+
const syncResult = await syncAllService(
|
|
200
|
+
config.collections,
|
|
201
|
+
store,
|
|
202
|
+
withContentTypeRules(
|
|
203
|
+
{
|
|
204
|
+
gitPull: syncOptions.gitPull,
|
|
205
|
+
runUpdateCmd: syncOptions.runUpdateCmd,
|
|
206
|
+
},
|
|
207
|
+
config
|
|
208
|
+
)
|
|
209
|
+
);
|
|
196
210
|
|
|
197
211
|
let embedResult: EmbedResult | null = null;
|
|
198
212
|
if (syncOptions.triggerEmbed !== false) {
|
package/src/serve/config-sync.ts
CHANGED
|
@@ -14,6 +14,7 @@ import {
|
|
|
14
14
|
type ApplyConfigResult,
|
|
15
15
|
type MutationResult,
|
|
16
16
|
} from "../core/config-mutation";
|
|
17
|
+
import { withContentTypeRules } from "../ingestion";
|
|
17
18
|
|
|
18
19
|
/**
|
|
19
20
|
* Apply a config mutation atomically with serialization.
|
|
@@ -53,7 +54,10 @@ export async function applyConfigChange(
|
|
|
53
54
|
);
|
|
54
55
|
|
|
55
56
|
if (result.ok) {
|
|
56
|
-
ctxHolder.watchService?.updateCollections(
|
|
57
|
+
ctxHolder.watchService?.updateCollections(
|
|
58
|
+
ctxHolder.config.collections,
|
|
59
|
+
withContentTypeRules({}, ctxHolder.config)
|
|
60
|
+
);
|
|
57
61
|
}
|
|
58
62
|
|
|
59
63
|
return result;
|
|
@@ -78,7 +82,10 @@ export async function applyConfigChangeTyped<T>(
|
|
|
78
82
|
);
|
|
79
83
|
|
|
80
84
|
if (result.ok) {
|
|
81
|
-
ctxHolder.watchService?.updateCollections(
|
|
85
|
+
ctxHolder.watchService?.updateCollections(
|
|
86
|
+
ctxHolder.config.collections,
|
|
87
|
+
withContentTypeRules({}, ctxHolder.config)
|
|
88
|
+
);
|
|
82
89
|
}
|
|
83
90
|
|
|
84
91
|
return result;
|
package/src/serve/routes/api.ts
CHANGED
|
@@ -82,7 +82,11 @@ import {
|
|
|
82
82
|
validateTag,
|
|
83
83
|
} from "../../core/tags";
|
|
84
84
|
import { validateRelPath } from "../../core/validation";
|
|
85
|
-
import {
|
|
85
|
+
import {
|
|
86
|
+
defaultSyncService,
|
|
87
|
+
type SyncResult,
|
|
88
|
+
withContentTypeRules,
|
|
89
|
+
} from "../../ingestion";
|
|
86
90
|
import { updateFrontmatterTags } from "../../ingestion/frontmatter";
|
|
87
91
|
import {
|
|
88
92
|
getCollectionEffectiveModels,
|
|
@@ -919,10 +923,17 @@ export async function handleCreateCollection(
|
|
|
919
923
|
return errorResponse("RUNTIME", "Collection not found after add", 500);
|
|
920
924
|
}
|
|
921
925
|
const jobResult = startJob("add", async (): Promise<SyncResult> => {
|
|
922
|
-
const result = await defaultSyncService.syncCollection(
|
|
923
|
-
|
|
924
|
-
|
|
925
|
-
|
|
926
|
+
const result = await defaultSyncService.syncCollection(
|
|
927
|
+
collection,
|
|
928
|
+
store,
|
|
929
|
+
withContentTypeRules(
|
|
930
|
+
{
|
|
931
|
+
gitPull: body.gitPull,
|
|
932
|
+
runUpdateCmd: true,
|
|
933
|
+
},
|
|
934
|
+
syncResult.config
|
|
935
|
+
)
|
|
936
|
+
);
|
|
926
937
|
if (result.filesAdded > 0 || result.filesUpdated > 0) {
|
|
927
938
|
if (ctxHolder.scheduler) {
|
|
928
939
|
await ctxHolder.scheduler.triggerNow();
|
|
@@ -1214,10 +1225,17 @@ export async function handleSync(
|
|
|
1214
1225
|
|
|
1215
1226
|
// Start background sync job
|
|
1216
1227
|
const jobResult = startJob("sync", async (): Promise<SyncResult> => {
|
|
1217
|
-
const result = await defaultSyncService.syncAll(
|
|
1218
|
-
|
|
1219
|
-
|
|
1220
|
-
|
|
1228
|
+
const result = await defaultSyncService.syncAll(
|
|
1229
|
+
collections,
|
|
1230
|
+
store,
|
|
1231
|
+
withContentTypeRules(
|
|
1232
|
+
{
|
|
1233
|
+
gitPull: body.gitPull,
|
|
1234
|
+
runUpdateCmd: true,
|
|
1235
|
+
},
|
|
1236
|
+
ctxHolder.config
|
|
1237
|
+
)
|
|
1238
|
+
);
|
|
1221
1239
|
if (result.totalFilesAdded > 0 || result.totalFilesUpdated > 0) {
|
|
1222
1240
|
if (ctxHolder.scheduler) {
|
|
1223
1241
|
await ctxHolder.scheduler.triggerNow();
|
|
@@ -1843,9 +1861,11 @@ export async function handleRenameDoc(
|
|
|
1843
1861
|
const nextUri = `gno://${collection.name}/${nextRelPath}`;
|
|
1844
1862
|
let warning: string | undefined;
|
|
1845
1863
|
try {
|
|
1846
|
-
await syncCollection(
|
|
1847
|
-
|
|
1848
|
-
|
|
1864
|
+
await syncCollection(
|
|
1865
|
+
collection,
|
|
1866
|
+
store,
|
|
1867
|
+
withContentTypeRules({ runUpdateCmd: false }, ctxHolder.config)
|
|
1868
|
+
);
|
|
1849
1869
|
} catch {
|
|
1850
1870
|
warning =
|
|
1851
1871
|
"File renamed on disk, but index refresh failed. Run Update All to reconcile the workspace.";
|
|
@@ -2063,9 +2083,11 @@ export async function handleTrashDoc(
|
|
|
2063
2083
|
}
|
|
2064
2084
|
let warning: string | undefined;
|
|
2065
2085
|
try {
|
|
2066
|
-
await syncCollection(
|
|
2067
|
-
|
|
2068
|
-
|
|
2086
|
+
await syncCollection(
|
|
2087
|
+
collection,
|
|
2088
|
+
store,
|
|
2089
|
+
withContentTypeRules({ runUpdateCmd: false }, ctxHolder.config)
|
|
2090
|
+
);
|
|
2069
2091
|
} catch {
|
|
2070
2092
|
warning =
|
|
2071
2093
|
"File moved to Trash, but index refresh failed. Run Update All to reconcile the workspace.";
|
|
@@ -2184,9 +2206,11 @@ export async function handleMoveDoc(
|
|
|
2184
2206
|
await renameFilePath(fullPath, nextFullPath);
|
|
2185
2207
|
let warning: string | undefined;
|
|
2186
2208
|
try {
|
|
2187
|
-
await defaultSyncService.syncCollection(
|
|
2188
|
-
|
|
2189
|
-
|
|
2209
|
+
await defaultSyncService.syncCollection(
|
|
2210
|
+
collection,
|
|
2211
|
+
store,
|
|
2212
|
+
withContentTypeRules({ runUpdateCmd: false }, ctxHolder.config)
|
|
2213
|
+
);
|
|
2190
2214
|
} catch {
|
|
2191
2215
|
warning =
|
|
2192
2216
|
"File moved on disk, but index refresh failed. Run Update All to reconcile the workspace.";
|
|
@@ -2305,9 +2329,11 @@ export async function handleDuplicateDoc(
|
|
|
2305
2329
|
await copyFilePath(fullPath, nextFullPath);
|
|
2306
2330
|
let warning: string | undefined;
|
|
2307
2331
|
try {
|
|
2308
|
-
await defaultSyncService.syncCollection(
|
|
2309
|
-
|
|
2310
|
-
|
|
2332
|
+
await defaultSyncService.syncCollection(
|
|
2333
|
+
collection,
|
|
2334
|
+
store,
|
|
2335
|
+
withContentTypeRules({ runUpdateCmd: false }, ctxHolder.config)
|
|
2336
|
+
);
|
|
2311
2337
|
} catch {
|
|
2312
2338
|
warning =
|
|
2313
2339
|
"File duplicated on disk, but index refresh failed. Run Update All to reconcile the workspace.";
|
|
@@ -2636,7 +2662,7 @@ export async function handleUpdateDoc(
|
|
|
2636
2662
|
const result = await defaultSyncService.syncCollection(
|
|
2637
2663
|
collection,
|
|
2638
2664
|
store,
|
|
2639
|
-
{ runUpdateCmd: false }
|
|
2665
|
+
withContentTypeRules({ runUpdateCmd: false }, ctxHolder.config)
|
|
2640
2666
|
);
|
|
2641
2667
|
// Notify scheduler after sync completes
|
|
2642
2668
|
ctxHolder.scheduler?.notifySyncComplete([doc.docid]);
|
|
@@ -2904,7 +2930,7 @@ export async function handleCreateCapture(
|
|
|
2904
2930
|
const result = await defaultSyncService.syncCollection(
|
|
2905
2931
|
collection,
|
|
2906
2932
|
store,
|
|
2907
|
-
{ runUpdateCmd: false }
|
|
2933
|
+
withContentTypeRules({ runUpdateCmd: false }, ctxHolder.config)
|
|
2908
2934
|
);
|
|
2909
2935
|
ctxHolder.scheduler?.notifySyncComplete([plan.relPath]);
|
|
2910
2936
|
ctxHolder.eventBus?.emit({
|
|
@@ -3140,7 +3166,7 @@ export async function handleCreateDoc(
|
|
|
3140
3166
|
const result = await defaultSyncService.syncCollection(
|
|
3141
3167
|
collection,
|
|
3142
3168
|
store,
|
|
3143
|
-
{ runUpdateCmd: false }
|
|
3169
|
+
withContentTypeRules({ runUpdateCmd: false }, ctxHolder.config)
|
|
3144
3170
|
);
|
|
3145
3171
|
// Notify scheduler after sync completes (use gnoUri as docid placeholder)
|
|
3146
3172
|
// The sync will create a proper docid, but we don't have it here yet
|
|
@@ -2,7 +2,7 @@ import { watch, type FSWatcher } from "node:fs";
|
|
|
2
2
|
import { join, normalize, sep } from "node:path";
|
|
3
3
|
|
|
4
4
|
import type { Collection } from "../config/types";
|
|
5
|
-
import type { CollectionSyncResult } from "../ingestion";
|
|
5
|
+
import type { CollectionSyncResult, SyncOptions } from "../ingestion";
|
|
6
6
|
import type { SqliteAdapter } from "../store/sqlite/adapter";
|
|
7
7
|
import type { DocumentEvent, DocumentEventBus } from "./doc-events";
|
|
8
8
|
import type { EmbedScheduler } from "./embed-scheduler";
|
|
@@ -39,6 +39,7 @@ interface CollectionWatchServiceOptions {
|
|
|
39
39
|
scheduler: EmbedScheduler | null;
|
|
40
40
|
eventBus?: DocumentEventBus | null;
|
|
41
41
|
callbacks?: CollectionWatchCallbacks;
|
|
42
|
+
syncOptions?: SyncOptions;
|
|
42
43
|
watchFactory?: typeof watch;
|
|
43
44
|
}
|
|
44
45
|
|
|
@@ -48,6 +49,7 @@ export class CollectionWatchService {
|
|
|
48
49
|
readonly #scheduler: EmbedScheduler | null;
|
|
49
50
|
readonly #eventBus: DocumentEventBus | null;
|
|
50
51
|
readonly #callbacks: CollectionWatchCallbacks | null;
|
|
52
|
+
#syncOptions: SyncOptions;
|
|
51
53
|
readonly #watchers = new Map<string, FSWatcher>();
|
|
52
54
|
readonly #pendingByCollection = new Map<string, Set<string>>();
|
|
53
55
|
readonly #timers = new Map<string, ReturnType<typeof setTimeout>>();
|
|
@@ -64,6 +66,7 @@ export class CollectionWatchService {
|
|
|
64
66
|
this.#scheduler = options.scheduler;
|
|
65
67
|
this.#eventBus = options.eventBus ?? null;
|
|
66
68
|
this.#callbacks = options.callbacks ?? null;
|
|
69
|
+
this.#syncOptions = options.syncOptions ?? {};
|
|
67
70
|
this.#watchFactory = options.watchFactory ?? watch;
|
|
68
71
|
}
|
|
69
72
|
|
|
@@ -71,8 +74,14 @@ export class CollectionWatchService {
|
|
|
71
74
|
this.updateCollections(this.#collections);
|
|
72
75
|
}
|
|
73
76
|
|
|
74
|
-
updateCollections(
|
|
77
|
+
updateCollections(
|
|
78
|
+
collections: Collection[],
|
|
79
|
+
syncOptions?: SyncOptions
|
|
80
|
+
): void {
|
|
75
81
|
this.#collections = collections;
|
|
82
|
+
if (syncOptions) {
|
|
83
|
+
this.#syncOptions = syncOptions;
|
|
84
|
+
}
|
|
76
85
|
const nextNames = new Set(collections.map((collection) => collection.name));
|
|
77
86
|
|
|
78
87
|
for (const [collectionName, watcher] of this.#watchers) {
|
|
@@ -204,6 +213,7 @@ export class CollectionWatchService {
|
|
|
204
213
|
collection,
|
|
205
214
|
this.#store,
|
|
206
215
|
{
|
|
216
|
+
...this.#syncOptions,
|
|
207
217
|
runUpdateCmd: false,
|
|
208
218
|
}
|
|
209
219
|
);
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Migration: content type rule fingerprint.
|
|
3
|
+
*
|
|
4
|
+
* @module src/store/migrations/009-content-type-rule-fingerprint
|
|
5
|
+
*/
|
|
6
|
+
|
|
7
|
+
import type { Migration } from "./runner";
|
|
8
|
+
|
|
9
|
+
export const migration: Migration = {
|
|
10
|
+
version: 9,
|
|
11
|
+
name: "content_type_rule_fingerprint",
|
|
12
|
+
|
|
13
|
+
up(db): void {
|
|
14
|
+
const columns = new Set(
|
|
15
|
+
db
|
|
16
|
+
.query<{ name: string }, []>("PRAGMA table_info(documents)")
|
|
17
|
+
.all()
|
|
18
|
+
.map((row) => row.name)
|
|
19
|
+
);
|
|
20
|
+
if (!columns.has("content_type_rules_fingerprint")) {
|
|
21
|
+
db.exec(
|
|
22
|
+
"ALTER TABLE documents ADD COLUMN content_type_rules_fingerprint TEXT"
|
|
23
|
+
);
|
|
24
|
+
}
|
|
25
|
+
db.exec(
|
|
26
|
+
"CREATE INDEX IF NOT EXISTS idx_documents_content_type_rules_fingerprint ON documents(content_type_rules_fingerprint)"
|
|
27
|
+
);
|
|
28
|
+
},
|
|
29
|
+
|
|
30
|
+
down(db): void {
|
|
31
|
+
db.exec(
|
|
32
|
+
"DROP INDEX IF EXISTS idx_documents_content_type_rules_fingerprint"
|
|
33
|
+
);
|
|
34
|
+
// SQLite cannot drop columns without a table rebuild; keep column on rollback.
|
|
35
|
+
},
|
|
36
|
+
};
|
|
@@ -22,6 +22,17 @@ import { migration as m005 } from "./005-graph-indexes";
|
|
|
22
22
|
import { migration as m006 } from "./006-document-metadata";
|
|
23
23
|
import { migration as m007 } from "./007-document-date-fields";
|
|
24
24
|
import { migration as m008 } from "./008-vector-fingerprints";
|
|
25
|
+
import { migration as m009 } from "./009-content-type-rule-fingerprint";
|
|
25
26
|
|
|
26
27
|
/** All migrations in order */
|
|
27
|
-
export const migrations = [
|
|
28
|
+
export const migrations = [
|
|
29
|
+
m001,
|
|
30
|
+
m002,
|
|
31
|
+
m003,
|
|
32
|
+
m004,
|
|
33
|
+
m005,
|
|
34
|
+
m006,
|
|
35
|
+
m007,
|
|
36
|
+
m008,
|
|
37
|
+
m009,
|
|
38
|
+
];
|
|
@@ -554,9 +554,10 @@ export class SqliteAdapter implements StorePort, SqliteDbProvider {
|
|
|
554
554
|
collection, rel_path, source_hash, source_mime, source_ext,
|
|
555
555
|
source_size, source_mtime, source_ctime, docid, uri, title, mirror_hash,
|
|
556
556
|
converter_id, converter_version, language_hint, content_type, categories,
|
|
557
|
-
author, frontmatter_date, date_fields,
|
|
558
|
-
|
|
559
|
-
|
|
557
|
+
author, frontmatter_date, date_fields, content_type_rules_fingerprint,
|
|
558
|
+
active, indexed_at, last_error_code, last_error_message, last_error_at,
|
|
559
|
+
ingest_version, updated_at
|
|
560
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 1, datetime('now'), ?, ?, ?, ?, datetime('now'))
|
|
560
561
|
ON CONFLICT(collection, rel_path) DO UPDATE SET
|
|
561
562
|
source_hash = excluded.source_hash,
|
|
562
563
|
source_mime = excluded.source_mime,
|
|
@@ -576,6 +577,7 @@ export class SqliteAdapter implements StorePort, SqliteDbProvider {
|
|
|
576
577
|
author = excluded.author,
|
|
577
578
|
frontmatter_date = excluded.frontmatter_date,
|
|
578
579
|
date_fields = excluded.date_fields,
|
|
580
|
+
content_type_rules_fingerprint = excluded.content_type_rules_fingerprint,
|
|
579
581
|
active = 1,
|
|
580
582
|
indexed_at = datetime('now'),
|
|
581
583
|
last_error_code = excluded.last_error_code,
|
|
@@ -605,6 +607,7 @@ export class SqliteAdapter implements StorePort, SqliteDbProvider {
|
|
|
605
607
|
doc.author ?? null,
|
|
606
608
|
doc.frontmatterDate ?? null,
|
|
607
609
|
doc.dateFields ? JSON.stringify(doc.dateFields) : null,
|
|
610
|
+
doc.contentTypeRulesFingerprint ?? null,
|
|
608
611
|
doc.lastErrorCode ?? null,
|
|
609
612
|
doc.lastErrorMessage ?? null,
|
|
610
613
|
doc.lastErrorCode ? new Date().toISOString() : null,
|
|
@@ -1366,7 +1369,9 @@ export class SqliteAdapter implements StorePort, SqliteDbProvider {
|
|
|
1366
1369
|
d.source_mtime,
|
|
1367
1370
|
d.frontmatter_date,
|
|
1368
1371
|
d.source_size,
|
|
1369
|
-
d.source_hash
|
|
1372
|
+
d.source_hash,
|
|
1373
|
+
d.content_type,
|
|
1374
|
+
d.categories
|
|
1370
1375
|
FROM fts_matches fm
|
|
1371
1376
|
JOIN documents d ON d.id = fm.rowid AND d.active = 1
|
|
1372
1377
|
WHERE 1 = 1
|
|
@@ -1392,6 +1397,8 @@ export class SqliteAdapter implements StorePort, SqliteDbProvider {
|
|
|
1392
1397
|
frontmatter_date: string | null;
|
|
1393
1398
|
source_size: number | null;
|
|
1394
1399
|
source_hash: string | null;
|
|
1400
|
+
content_type: string | null;
|
|
1401
|
+
categories: string | null;
|
|
1395
1402
|
}
|
|
1396
1403
|
|
|
1397
1404
|
const queryParams = [builtQuery.query, ftsLimit, ...params];
|
|
@@ -1416,6 +1423,8 @@ export class SqliteAdapter implements StorePort, SqliteDbProvider {
|
|
|
1416
1423
|
frontmatterDate: r.frontmatter_date ?? undefined,
|
|
1417
1424
|
sourceSize: r.source_size ?? undefined,
|
|
1418
1425
|
sourceHash: r.source_hash ?? undefined,
|
|
1426
|
+
contentType: r.content_type ?? undefined,
|
|
1427
|
+
categories: parseCategoriesJson(r.categories) ?? undefined,
|
|
1419
1428
|
}))
|
|
1420
1429
|
);
|
|
1421
1430
|
} catch (cause) {
|
|
@@ -3522,6 +3531,7 @@ interface DbDocumentRow {
|
|
|
3522
3531
|
author: string | null;
|
|
3523
3532
|
frontmatter_date: string | null;
|
|
3524
3533
|
date_fields: string | null;
|
|
3534
|
+
content_type_rules_fingerprint: string | null;
|
|
3525
3535
|
indexed_at: string | null;
|
|
3526
3536
|
active: number;
|
|
3527
3537
|
ingest_version: number | null;
|
|
@@ -3580,19 +3590,23 @@ function mapContextRow(row: DbContextRow): ContextRow {
|
|
|
3580
3590
|
};
|
|
3581
3591
|
}
|
|
3582
3592
|
|
|
3583
|
-
function
|
|
3584
|
-
|
|
3585
|
-
|
|
3586
|
-
|
|
3587
|
-
|
|
3588
|
-
|
|
3589
|
-
|
|
3590
|
-
|
|
3591
|
-
} catch {
|
|
3592
|
-
categories = null;
|
|
3593
|
+
function parseCategoriesJson(raw: string | null): string[] | null {
|
|
3594
|
+
if (!raw) {
|
|
3595
|
+
return null;
|
|
3596
|
+
}
|
|
3597
|
+
try {
|
|
3598
|
+
const parsed = JSON.parse(raw);
|
|
3599
|
+
if (Array.isArray(parsed)) {
|
|
3600
|
+
return parsed.filter((v): v is string => typeof v === "string");
|
|
3593
3601
|
}
|
|
3602
|
+
} catch {
|
|
3603
|
+
return null;
|
|
3594
3604
|
}
|
|
3605
|
+
return null;
|
|
3606
|
+
}
|
|
3595
3607
|
|
|
3608
|
+
function mapDocumentRow(row: DbDocumentRow): DocumentRow {
|
|
3609
|
+
const categories = parseCategoriesJson(row.categories);
|
|
3596
3610
|
let dateFields: Record<string, string> | null = null;
|
|
3597
3611
|
if (row.date_fields) {
|
|
3598
3612
|
try {
|
|
@@ -3638,6 +3652,7 @@ function mapDocumentRow(row: DbDocumentRow): DocumentRow {
|
|
|
3638
3652
|
indexedAt: row.indexed_at,
|
|
3639
3653
|
active: row.active === 1,
|
|
3640
3654
|
ingestVersion: row.ingest_version,
|
|
3655
|
+
contentTypeRulesFingerprint: row.content_type_rules_fingerprint,
|
|
3641
3656
|
lastErrorCode: row.last_error_code,
|
|
3642
3657
|
lastErrorMessage: row.last_error_message,
|
|
3643
3658
|
lastErrorAt: row.last_error_at,
|
package/src/store/types.ts
CHANGED
|
@@ -118,6 +118,8 @@ export interface DocumentRow {
|
|
|
118
118
|
active: boolean;
|
|
119
119
|
/** Ingest schema version for backfill detection */
|
|
120
120
|
ingestVersion: number | null;
|
|
121
|
+
/** Fingerprint of normalized content type rules used for derived metadata */
|
|
122
|
+
contentTypeRulesFingerprint?: string | null;
|
|
121
123
|
|
|
122
124
|
// Error tracking
|
|
123
125
|
lastErrorCode: string | null;
|
|
@@ -264,6 +266,8 @@ export interface DocumentInput {
|
|
|
264
266
|
lastErrorMessage?: string;
|
|
265
267
|
/** Ingest schema version for backfill detection */
|
|
266
268
|
ingestVersion?: number;
|
|
269
|
+
/** Fingerprint of normalized content type rules used for derived metadata */
|
|
270
|
+
contentTypeRulesFingerprint?: string;
|
|
267
271
|
}
|
|
268
272
|
|
|
269
273
|
/** Result of upserting a document */
|
|
@@ -345,6 +349,8 @@ export interface FtsResult {
|
|
|
345
349
|
frontmatterDate?: string;
|
|
346
350
|
sourceSize?: number;
|
|
347
351
|
sourceHash?: string;
|
|
352
|
+
contentType?: string;
|
|
353
|
+
categories?: string[];
|
|
348
354
|
}
|
|
349
355
|
|
|
350
356
|
// ─────────────────────────────────────────────────────────────────────────────
|