@gmickel/gno 1.7.1 → 1.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +10 -1
- package/assets/skill/SKILL.md +41 -0
- package/assets/skill/cli-reference.md +34 -0
- package/assets/skill/examples.md +37 -0
- package/assets/skill/mcp-reference.md +21 -0
- package/package.json +1 -1
- package/src/cli/commands/capture.ts +282 -0
- package/src/cli/commands/index-cmd.ts +17 -6
- package/src/cli/commands/shared.ts +17 -1
- package/src/cli/commands/update.ts +17 -6
- package/src/cli/options.ts +2 -0
- package/src/cli/program.ts +64 -0
- package/src/config/content-types.ts +140 -0
- package/src/config/defaults.ts +1 -0
- package/src/config/index.ts +14 -0
- package/src/config/loader.ts +11 -2
- package/src/config/types.ts +37 -1
- package/src/core/capture-write.ts +38 -0
- package/src/core/capture.ts +746 -0
- package/src/core/config-mutation.ts +14 -2
- package/src/core/file-ops.ts +21 -2
- package/src/core/note-presets.ts +61 -5
- package/src/ingestion/frontmatter.ts +77 -2
- package/src/ingestion/index.ts +2 -0
- package/src/ingestion/sync-options.ts +29 -0
- package/src/ingestion/sync.ts +104 -16
- package/src/ingestion/types.ts +14 -0
- package/src/mcp/tools/add-collection.ts +8 -5
- package/src/mcp/tools/capture.ts +137 -191
- package/src/mcp/tools/index-cmd.ts +13 -7
- package/src/mcp/tools/index.ts +43 -9
- package/src/mcp/tools/sync.ts +12 -6
- package/src/mcp/tools/workspace-write.ts +16 -10
- package/src/pipeline/hybrid.ts +2 -0
- package/src/pipeline/search.ts +2 -0
- package/src/pipeline/types.ts +2 -0
- package/src/pipeline/vsearch.ts +4 -0
- package/src/sdk/client.ts +156 -20
- package/src/sdk/index.ts +2 -0
- package/src/sdk/types.ts +6 -0
- package/src/serve/background-runtime.ts +18 -4
- package/src/serve/config-sync.ts +9 -2
- package/src/serve/public/components/CaptureModal.tsx +248 -10
- package/src/serve/public/globals.built.css +1 -1
- package/src/serve/routes/api.ts +238 -26
- package/src/serve/server.ts +12 -0
- package/src/serve/watch-service.ts +12 -2
- package/src/store/migrations/009-content-type-rule-fingerprint.ts +36 -0
- package/src/store/migrations/index.ts +12 -1
- package/src/store/sqlite/adapter.ts +29 -14
- package/src/store/types.ts +6 -0
|
@@ -7,7 +7,12 @@
|
|
|
7
7
|
import type { Config } from "../config/types";
|
|
8
8
|
import type { SqliteAdapter } from "../store/sqlite/adapter";
|
|
9
9
|
|
|
10
|
-
import {
|
|
10
|
+
import {
|
|
11
|
+
formatConfigWarnings,
|
|
12
|
+
loadConfig,
|
|
13
|
+
normalizeConfigContentTypes,
|
|
14
|
+
saveConfig,
|
|
15
|
+
} from "../config";
|
|
11
16
|
|
|
12
17
|
export interface ConfigMutationContext {
|
|
13
18
|
store: SqliteAdapter;
|
|
@@ -53,6 +58,9 @@ export async function applyConfigChange<T = void>(
|
|
|
53
58
|
code: "LOAD_ERROR",
|
|
54
59
|
};
|
|
55
60
|
}
|
|
61
|
+
for (const warning of formatConfigWarnings(loadResult.warnings)) {
|
|
62
|
+
console.warn(warning);
|
|
63
|
+
}
|
|
56
64
|
|
|
57
65
|
const mutationResult = await mutate(loadResult.value);
|
|
58
66
|
if (!mutationResult.ok) {
|
|
@@ -63,7 +71,11 @@ export async function applyConfigChange<T = void>(
|
|
|
63
71
|
};
|
|
64
72
|
}
|
|
65
73
|
|
|
66
|
-
const
|
|
74
|
+
const normalized = normalizeConfigContentTypes(mutationResult.config);
|
|
75
|
+
for (const warning of formatConfigWarnings(normalized.warnings)) {
|
|
76
|
+
console.warn(warning);
|
|
77
|
+
}
|
|
78
|
+
const newConfig = normalized.config;
|
|
67
79
|
const saveResult = await saveConfig(newConfig, ctx.configPath);
|
|
68
80
|
if (!saveResult.ok) {
|
|
69
81
|
return {
|
package/src/core/file-ops.ts
CHANGED
|
@@ -4,8 +4,8 @@
|
|
|
4
4
|
* @module src/core/file-ops
|
|
5
5
|
*/
|
|
6
6
|
|
|
7
|
-
// node:fs/promises for rename/unlink (no Bun equivalent for structure ops)
|
|
8
|
-
import { copyFile, mkdir, rename, unlink } from "node:fs/promises";
|
|
7
|
+
// node:fs/promises for rename/unlink/link (no Bun equivalent for structure ops or exclusive hard-link create)
|
|
8
|
+
import { copyFile, link, mkdir, rename, unlink } from "node:fs/promises";
|
|
9
9
|
// node:os platform/homedir/tmpdir: no Bun equivalent
|
|
10
10
|
import { homedir, platform as getPlatform, tmpdir } from "node:os";
|
|
11
11
|
// node:path dirname/join/parse: no Bun equivalent
|
|
@@ -27,6 +27,25 @@ export async function atomicWrite(
|
|
|
27
27
|
}
|
|
28
28
|
}
|
|
29
29
|
|
|
30
|
+
export async function atomicCreate(
|
|
31
|
+
path: string,
|
|
32
|
+
content: string
|
|
33
|
+
): Promise<void> {
|
|
34
|
+
const tempPath = `${path}.tmp.${crypto.randomUUID()}`;
|
|
35
|
+
await Bun.write(tempPath, content);
|
|
36
|
+
try {
|
|
37
|
+
await link(tempPath, path);
|
|
38
|
+
} catch (e) {
|
|
39
|
+
await unlink(tempPath).catch(() => {
|
|
40
|
+
/* ignore cleanup errors */
|
|
41
|
+
});
|
|
42
|
+
throw e;
|
|
43
|
+
}
|
|
44
|
+
await unlink(tempPath).catch(() => {
|
|
45
|
+
/* ignore cleanup errors */
|
|
46
|
+
});
|
|
47
|
+
}
|
|
48
|
+
|
|
30
49
|
async function runCommand(cmd: string[]): Promise<void> {
|
|
31
50
|
const proc = Bun.spawn({
|
|
32
51
|
cmd,
|
package/src/core/note-presets.ts
CHANGED
|
@@ -14,7 +14,11 @@ export type NotePresetId =
|
|
|
14
14
|
| "research-note"
|
|
15
15
|
| "decision-note"
|
|
16
16
|
| "prompt-pattern"
|
|
17
|
-
| "source-summary"
|
|
17
|
+
| "source-summary"
|
|
18
|
+
| "idea-original"
|
|
19
|
+
| "person"
|
|
20
|
+
| "company-project"
|
|
21
|
+
| "meeting";
|
|
18
22
|
|
|
19
23
|
export interface NotePresetDefinition {
|
|
20
24
|
id: NotePresetId;
|
|
@@ -104,14 +108,14 @@ export const NOTE_PRESETS: NotePresetDefinition[] = [
|
|
|
104
108
|
{
|
|
105
109
|
id: "decision-note",
|
|
106
110
|
label: "Decision Note",
|
|
107
|
-
description: "
|
|
111
|
+
description: "Current synthesis, decision, rationale, and timeline.",
|
|
108
112
|
defaultTags: ["decision"],
|
|
109
113
|
frontmatter: {
|
|
110
114
|
category: "decision",
|
|
111
115
|
status: "proposed",
|
|
112
116
|
},
|
|
113
117
|
body: (title) =>
|
|
114
|
-
`# ${title}\n\n##
|
|
118
|
+
`# ${title}\n\n## Current Synthesis\n\n## Decision\n\n## Rationale\n\n## Consequences\n\n## Timeline\n`,
|
|
115
119
|
},
|
|
116
120
|
{
|
|
117
121
|
id: "prompt-pattern",
|
|
@@ -127,14 +131,66 @@ export const NOTE_PRESETS: NotePresetDefinition[] = [
|
|
|
127
131
|
{
|
|
128
132
|
id: "source-summary",
|
|
129
133
|
label: "Source Summary",
|
|
130
|
-
description: "Summarize
|
|
134
|
+
description: "Summarize a source with synthesis above evidence timeline.",
|
|
131
135
|
defaultTags: ["source", "summary"],
|
|
132
136
|
frontmatter: {
|
|
133
137
|
category: "source-summary",
|
|
134
138
|
sources: [],
|
|
135
139
|
},
|
|
136
140
|
body: (title) =>
|
|
137
|
-
`# ${title}\n\n##
|
|
141
|
+
`# ${title}\n\n## Current Synthesis\n\n## Important Claims\n\n## Assessment\n\n## Timeline\n\n## Evidence / Quotes\n`,
|
|
142
|
+
},
|
|
143
|
+
{
|
|
144
|
+
id: "idea-original",
|
|
145
|
+
label: "Original Idea",
|
|
146
|
+
description:
|
|
147
|
+
"Capture exact phrasing, context, related concepts, and publish potential.",
|
|
148
|
+
frontmatter: {
|
|
149
|
+
type: "idea-original",
|
|
150
|
+
category: "idea-original",
|
|
151
|
+
tags: [],
|
|
152
|
+
},
|
|
153
|
+
body: (title) =>
|
|
154
|
+
`# ${title}\n\n## Current Synthesis\n\n## Open Threads\n\n## Assessment\n\n## Timeline\n\n## Exact Phrasing\n\n## Related Concepts\n`,
|
|
155
|
+
},
|
|
156
|
+
{
|
|
157
|
+
id: "person",
|
|
158
|
+
label: "Person",
|
|
159
|
+
description:
|
|
160
|
+
"Track current state, relationship, assessment, open threads, and timeline.",
|
|
161
|
+
frontmatter: {
|
|
162
|
+
type: "person",
|
|
163
|
+
category: "person",
|
|
164
|
+
tags: [],
|
|
165
|
+
},
|
|
166
|
+
body: (title) =>
|
|
167
|
+
`# ${title}\n\n## Current Synthesis\n\n## Open Threads\n\n## Assessment\n\n## Timeline\n\n## Relationship\n`,
|
|
168
|
+
},
|
|
169
|
+
{
|
|
170
|
+
id: "company-project",
|
|
171
|
+
label: "Company / Project",
|
|
172
|
+
description:
|
|
173
|
+
"Track state, changes, decisions, people, open threads, and timeline.",
|
|
174
|
+
frontmatter: {
|
|
175
|
+
type: "company-project",
|
|
176
|
+
category: "company-project",
|
|
177
|
+
tags: [],
|
|
178
|
+
},
|
|
179
|
+
body: (title) =>
|
|
180
|
+
`# ${title}\n\n## Current Synthesis\n\n## Open Threads\n\n## Assessment\n\n## Timeline\n\n## Decisions\n\n## People\n`,
|
|
181
|
+
},
|
|
182
|
+
{
|
|
183
|
+
id: "meeting",
|
|
184
|
+
label: "Meeting",
|
|
185
|
+
description:
|
|
186
|
+
"Put analysis above the timeline; keep transcript, notes, and actions below.",
|
|
187
|
+
frontmatter: {
|
|
188
|
+
type: "meeting",
|
|
189
|
+
category: "meeting",
|
|
190
|
+
tags: [],
|
|
191
|
+
},
|
|
192
|
+
body: (title) =>
|
|
193
|
+
`# ${title}\n\n## Current Synthesis\n\n## Open Threads\n\n## Assessment\n\n## Timeline\n\n## Transcript / Notes\n\n## Action Items\n`,
|
|
138
194
|
},
|
|
139
195
|
];
|
|
140
196
|
|
|
@@ -117,6 +117,78 @@ function parseTagsValue(
|
|
|
117
117
|
return [];
|
|
118
118
|
}
|
|
119
119
|
|
|
120
|
+
function parseInlineArrayValue(value: string): string[] | undefined {
|
|
121
|
+
const inlineMatch = INLINE_ARRAY_REGEX.exec(value.trim());
|
|
122
|
+
if (!inlineMatch) {
|
|
123
|
+
return undefined;
|
|
124
|
+
}
|
|
125
|
+
const rawItems = inlineMatch[1];
|
|
126
|
+
if (rawItems === undefined) {
|
|
127
|
+
return [];
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
return rawItems
|
|
131
|
+
.split(",")
|
|
132
|
+
.map((item) => unquoteScalar(item.trim()))
|
|
133
|
+
.filter((item) => item.length > 0);
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
function unquoteScalar(value: string): string {
|
|
137
|
+
const trimmed = value.trim();
|
|
138
|
+
if (trimmed.length < 2) {
|
|
139
|
+
return trimmed;
|
|
140
|
+
}
|
|
141
|
+
const first = trimmed[0];
|
|
142
|
+
const last = trimmed.at(-1);
|
|
143
|
+
if ((first === '"' && last === '"') || (first === "'" && last === "'")) {
|
|
144
|
+
return trimmed.slice(1, -1).trim();
|
|
145
|
+
}
|
|
146
|
+
return trimmed;
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
function parseBlockArrayValue(lines: string[], startIdx: number): string[] {
|
|
150
|
+
const items: string[] = [];
|
|
151
|
+
for (let i = startIdx + 1; i < lines.length; i++) {
|
|
152
|
+
const line = lines[i];
|
|
153
|
+
if (line === undefined) break;
|
|
154
|
+
const itemMatch = YAML_ARRAY_ITEM_REGEX.exec(line);
|
|
155
|
+
if (itemMatch?.[1]) {
|
|
156
|
+
const item = unquoteScalar(itemMatch[1]);
|
|
157
|
+
if (item.length > 0) {
|
|
158
|
+
items.push(item);
|
|
159
|
+
}
|
|
160
|
+
continue;
|
|
161
|
+
}
|
|
162
|
+
if (
|
|
163
|
+
line.trim().length > 0 &&
|
|
164
|
+
!line.startsWith(" ") &&
|
|
165
|
+
!line.startsWith("\t")
|
|
166
|
+
) {
|
|
167
|
+
break;
|
|
168
|
+
}
|
|
169
|
+
}
|
|
170
|
+
return items;
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
function parseMetadataValue(
|
|
174
|
+
value: string,
|
|
175
|
+
lines: string[],
|
|
176
|
+
startIdx: number
|
|
177
|
+
): string | string[] | undefined {
|
|
178
|
+
const trimmed = value.trim();
|
|
179
|
+
const inlineArray = parseInlineArrayValue(trimmed);
|
|
180
|
+
if (inlineArray) {
|
|
181
|
+
return inlineArray;
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
if (trimmed.length === 0) {
|
|
185
|
+
const blockArray = parseBlockArrayValue(lines, startIdx);
|
|
186
|
+
return blockArray.length > 0 ? blockArray : undefined;
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
return unquoteScalar(trimmed);
|
|
190
|
+
}
|
|
191
|
+
|
|
120
192
|
/**
|
|
121
193
|
* Parse frontmatter from markdown source.
|
|
122
194
|
* Returns extracted tags and metadata.
|
|
@@ -170,8 +242,11 @@ export function parseFrontmatter(source: string): FrontmatterResult {
|
|
|
170
242
|
if (colonIdx > 0) {
|
|
171
243
|
const key = line.slice(0, colonIdx).trim();
|
|
172
244
|
const value = line.slice(colonIdx + 1).trim();
|
|
173
|
-
if (key !== "tags"
|
|
174
|
-
|
|
245
|
+
if (key !== "tags") {
|
|
246
|
+
const metadataValue = parseMetadataValue(value, lines, i);
|
|
247
|
+
if (metadataValue !== undefined) {
|
|
248
|
+
result.metadata[key] = metadataValue;
|
|
249
|
+
}
|
|
175
250
|
}
|
|
176
251
|
}
|
|
177
252
|
}
|
package/src/ingestion/index.ts
CHANGED
|
@@ -10,12 +10,14 @@ export { defaultChunker, MarkdownChunker } from "./chunker";
|
|
|
10
10
|
export { defaultLanguageDetector, SimpleLanguageDetector } from "./language";
|
|
11
11
|
// Sync service
|
|
12
12
|
export { defaultSyncService, SyncService } from "./sync";
|
|
13
|
+
export { resolveContentTypeRules, withContentTypeRules } from "./sync-options";
|
|
13
14
|
// Types
|
|
14
15
|
export type {
|
|
15
16
|
ChunkerPort,
|
|
16
17
|
ChunkOutput,
|
|
17
18
|
ChunkParams,
|
|
18
19
|
CollectionSyncResult,
|
|
20
|
+
ContentTypeSource,
|
|
19
21
|
FileSyncResult,
|
|
20
22
|
FileSyncStatus,
|
|
21
23
|
LanguageDetectorPort,
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Sync option helpers.
|
|
3
|
+
*
|
|
4
|
+
* @module src/ingestion/sync-options
|
|
5
|
+
*/
|
|
6
|
+
|
|
7
|
+
import type { Config, NormalizedContentTypeRule } from "../config";
|
|
8
|
+
import type { SyncOptions } from "./types";
|
|
9
|
+
|
|
10
|
+
import { fingerprintContentTypeRules, normalizeContentTypes } from "../config";
|
|
11
|
+
|
|
12
|
+
export function resolveContentTypeRules(
|
|
13
|
+
config?: Pick<Config, "contentTypes">
|
|
14
|
+
): NormalizedContentTypeRule[] {
|
|
15
|
+
return normalizeContentTypes(config?.contentTypes ?? []).rules;
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
export function withContentTypeRules(
|
|
19
|
+
options: SyncOptions = {},
|
|
20
|
+
config?: Pick<Config, "contentTypes">
|
|
21
|
+
): SyncOptions {
|
|
22
|
+
const rules = options.contentTypeRules ?? resolveContentTypeRules(config);
|
|
23
|
+
return {
|
|
24
|
+
...options,
|
|
25
|
+
contentTypeRules: rules,
|
|
26
|
+
contentTypeRulesFingerprint:
|
|
27
|
+
options.contentTypeRulesFingerprint ?? fingerprintContentTypeRules(rules),
|
|
28
|
+
};
|
|
29
|
+
}
|
package/src/ingestion/sync.ts
CHANGED
|
@@ -10,6 +10,7 @@ import { stat } from "node:fs/promises";
|
|
|
10
10
|
// node:path for join (no Bun path utils)
|
|
11
11
|
import { join } from "node:path";
|
|
12
12
|
|
|
13
|
+
import type { NormalizedContentTypeRule } from "../config";
|
|
13
14
|
import type { Collection } from "../config/types";
|
|
14
15
|
import type {
|
|
15
16
|
ChunkInput,
|
|
@@ -22,6 +23,7 @@ import type {
|
|
|
22
23
|
import type {
|
|
23
24
|
ChunkerPort,
|
|
24
25
|
CollectionSyncResult,
|
|
26
|
+
ContentTypeSource,
|
|
25
27
|
FileSyncResult,
|
|
26
28
|
ProcessDecision,
|
|
27
29
|
SyncOptions,
|
|
@@ -30,6 +32,7 @@ import type {
|
|
|
30
32
|
WalkerPort,
|
|
31
33
|
} from "./types";
|
|
32
34
|
|
|
35
|
+
import { fingerprintContentTypeRules } from "../config";
|
|
33
36
|
import { getDefaultMimeDetector, type MimeDetector } from "../converters/mime";
|
|
34
37
|
import {
|
|
35
38
|
type ConversionPipeline,
|
|
@@ -68,6 +71,7 @@ const MAX_CONCURRENCY = 16;
|
|
|
68
71
|
* Documents with ingestVersion < INGEST_VERSION will be re-processed.
|
|
69
72
|
*/
|
|
70
73
|
export const INGEST_VERSION = 5;
|
|
74
|
+
const EMPTY_CONTENT_TYPE_RULES_FINGERPRINT = fingerprintContentTypeRules([]);
|
|
71
75
|
|
|
72
76
|
/**
|
|
73
77
|
* Decide whether to process a file or skip it.
|
|
@@ -76,7 +80,8 @@ export const INGEST_VERSION = 5;
|
|
|
76
80
|
*/
|
|
77
81
|
function decideAction(
|
|
78
82
|
existing: DocumentRow | null,
|
|
79
|
-
sourceHash: string
|
|
83
|
+
sourceHash: string,
|
|
84
|
+
contentTypeRulesFingerprint: string
|
|
80
85
|
): ProcessDecision {
|
|
81
86
|
// No existing doc - must process
|
|
82
87
|
if (!existing) {
|
|
@@ -108,6 +113,16 @@ function decideAction(
|
|
|
108
113
|
return { kind: "repair", reason: "ingest version outdated" };
|
|
109
114
|
}
|
|
110
115
|
|
|
116
|
+
const hasLegacyEmptyRulesFingerprint =
|
|
117
|
+
existing.contentTypeRulesFingerprint === null &&
|
|
118
|
+
contentTypeRulesFingerprint === EMPTY_CONTENT_TYPE_RULES_FINGERPRINT;
|
|
119
|
+
if (
|
|
120
|
+
existing.contentTypeRulesFingerprint !== contentTypeRulesFingerprint &&
|
|
121
|
+
!hasLegacyEmptyRulesFingerprint
|
|
122
|
+
) {
|
|
123
|
+
return { kind: "repair", reason: "content type rules changed" };
|
|
124
|
+
}
|
|
125
|
+
|
|
111
126
|
// All good - skip
|
|
112
127
|
return { kind: "skip", reason: "unchanged" };
|
|
113
128
|
}
|
|
@@ -143,6 +158,7 @@ function extractTags(markdown: string): string[] {
|
|
|
143
158
|
|
|
144
159
|
interface DocumentMetadata {
|
|
145
160
|
contentType?: string;
|
|
161
|
+
contentTypeSource: ContentTypeSource;
|
|
146
162
|
categories?: string[];
|
|
147
163
|
author?: string;
|
|
148
164
|
frontmatterDate?: string;
|
|
@@ -210,47 +226,95 @@ function normalizeDate(value: unknown): string | undefined {
|
|
|
210
226
|
return parsed.toISOString();
|
|
211
227
|
}
|
|
212
228
|
|
|
213
|
-
function
|
|
229
|
+
function inferPathContentType(
|
|
230
|
+
relPath: string,
|
|
231
|
+
ext: string
|
|
232
|
+
): {
|
|
233
|
+
contentType: string;
|
|
234
|
+
source: ContentTypeSource;
|
|
235
|
+
} {
|
|
214
236
|
const lowerPath = relPath.toLowerCase();
|
|
215
237
|
if (CODE_EXTENSIONS.has(ext.toLowerCase())) {
|
|
216
|
-
return "code";
|
|
238
|
+
return { contentType: "code", source: "path-ext" };
|
|
217
239
|
}
|
|
218
240
|
if (/(meeting|standup|retro|minutes)/.test(lowerPath)) {
|
|
219
|
-
return "meeting";
|
|
241
|
+
return { contentType: "meeting", source: "path-ext" };
|
|
220
242
|
}
|
|
221
243
|
if (/(spec|rfc|adr|design)/.test(lowerPath)) {
|
|
222
|
-
return "spec";
|
|
244
|
+
return { contentType: "spec", source: "path-ext" };
|
|
223
245
|
}
|
|
224
246
|
if (/(notes|journal|log)/.test(lowerPath)) {
|
|
225
|
-
return "notes";
|
|
247
|
+
return { contentType: "notes", source: "path-ext" };
|
|
248
|
+
}
|
|
249
|
+
return { contentType: "prose", source: "fallback" };
|
|
250
|
+
}
|
|
251
|
+
|
|
252
|
+
function normalizeFrontmatterScalar(value: string): string {
|
|
253
|
+
const trimmed = value.trim();
|
|
254
|
+
if (trimmed.length < 2) {
|
|
255
|
+
return trimmed;
|
|
256
|
+
}
|
|
257
|
+
const first = trimmed[0];
|
|
258
|
+
const last = trimmed.at(-1);
|
|
259
|
+
if ((first === '"' && last === '"') || (first === "'" && last === "'")) {
|
|
260
|
+
return trimmed.slice(1, -1).trim();
|
|
226
261
|
}
|
|
227
|
-
return
|
|
262
|
+
return trimmed;
|
|
228
263
|
}
|
|
229
264
|
|
|
230
265
|
function parseCategories(input: unknown): string[] {
|
|
231
266
|
if (Array.isArray(input)) {
|
|
232
267
|
return input
|
|
233
268
|
.filter((v): v is string => typeof v === "string")
|
|
234
|
-
.map((v) => v
|
|
269
|
+
.map((v) => normalizeFrontmatterScalar(v).toLowerCase())
|
|
235
270
|
.filter((v) => v.length > 0);
|
|
236
271
|
}
|
|
237
272
|
if (typeof input === "string") {
|
|
238
273
|
return input
|
|
239
274
|
.split(",")
|
|
240
|
-
.map((v) => v
|
|
275
|
+
.map((v) => normalizeFrontmatterScalar(v).toLowerCase())
|
|
241
276
|
.filter((v) => v.length > 0);
|
|
242
277
|
}
|
|
243
278
|
return [];
|
|
244
279
|
}
|
|
245
280
|
|
|
246
|
-
function
|
|
281
|
+
function matchPrefixContentType(
|
|
282
|
+
relPath: string,
|
|
283
|
+
rules: NormalizedContentTypeRule[]
|
|
284
|
+
): string | undefined {
|
|
285
|
+
for (const rule of rules) {
|
|
286
|
+
if (rule.prefixes.some((prefix) => relPath.startsWith(prefix))) {
|
|
287
|
+
return rule.id;
|
|
288
|
+
}
|
|
289
|
+
}
|
|
290
|
+
return undefined;
|
|
291
|
+
}
|
|
292
|
+
|
|
293
|
+
export function extractDocumentMetadata(
|
|
247
294
|
markdown: string,
|
|
248
295
|
relPath: string,
|
|
249
|
-
ext: string
|
|
296
|
+
ext: string,
|
|
297
|
+
contentTypeRules: NormalizedContentTypeRule[] = []
|
|
250
298
|
): DocumentMetadata {
|
|
251
299
|
const parsed = parseFrontmatter(markdown);
|
|
252
300
|
const metadata = parsed.metadata;
|
|
253
|
-
const
|
|
301
|
+
const typedRules = new Map(contentTypeRules.map((rule) => [rule.id, rule]));
|
|
302
|
+
const rawFrontmatterType =
|
|
303
|
+
typeof metadata.type === "string"
|
|
304
|
+
? normalizeFrontmatterScalar(metadata.type)
|
|
305
|
+
: "";
|
|
306
|
+
const frontmatterType = typedRules.get(rawFrontmatterType)?.id;
|
|
307
|
+
const prefixType =
|
|
308
|
+
frontmatterType === undefined
|
|
309
|
+
? matchPrefixContentType(relPath, contentTypeRules)
|
|
310
|
+
: undefined;
|
|
311
|
+
const inferred = inferPathContentType(relPath, ext);
|
|
312
|
+
const contentType = frontmatterType ?? prefixType ?? inferred.contentType;
|
|
313
|
+
const contentTypeSource: ContentTypeSource = frontmatterType
|
|
314
|
+
? "frontmatter-type"
|
|
315
|
+
: prefixType
|
|
316
|
+
? "prefix"
|
|
317
|
+
: inferred.source;
|
|
254
318
|
const categories = new Set<string>([contentType]);
|
|
255
319
|
|
|
256
320
|
const fmCategories = parseCategories(
|
|
@@ -299,6 +363,7 @@ function extractDocumentMetadata(
|
|
|
299
363
|
|
|
300
364
|
return {
|
|
301
365
|
contentType,
|
|
366
|
+
contentTypeSource,
|
|
302
367
|
categories: [...categories],
|
|
303
368
|
author,
|
|
304
369
|
frontmatterDate,
|
|
@@ -488,6 +553,10 @@ export class SyncService {
|
|
|
488
553
|
const hasher = new Bun.CryptoHasher("sha256");
|
|
489
554
|
hasher.update(bytes);
|
|
490
555
|
const sourceHash = hasher.digest("hex");
|
|
556
|
+
const contentTypeRules = options.contentTypeRules ?? [];
|
|
557
|
+
const contentTypeRulesFingerprint =
|
|
558
|
+
options.contentTypeRulesFingerprint ??
|
|
559
|
+
fingerprintContentTypeRules(contentTypeRules);
|
|
491
560
|
|
|
492
561
|
// 4. Check existing doc for skip/repair decision
|
|
493
562
|
const existingResult = await store.getDocument(
|
|
@@ -495,7 +564,11 @@ export class SyncService {
|
|
|
495
564
|
entry.relPath
|
|
496
565
|
);
|
|
497
566
|
const existing = existingResult.ok ? existingResult.value : null;
|
|
498
|
-
const decision = decideAction(
|
|
567
|
+
const decision = decideAction(
|
|
568
|
+
existing,
|
|
569
|
+
sourceHash,
|
|
570
|
+
contentTypeRulesFingerprint
|
|
571
|
+
);
|
|
499
572
|
|
|
500
573
|
if (decision.kind === "skip") {
|
|
501
574
|
return { relPath: entry.relPath, status: "unchanged" };
|
|
@@ -565,7 +638,8 @@ export class SyncService {
|
|
|
565
638
|
const extractedMetadata = extractDocumentMetadata(
|
|
566
639
|
artifact.markdown,
|
|
567
640
|
entry.relPath,
|
|
568
|
-
mime.ext
|
|
641
|
+
mime.ext,
|
|
642
|
+
contentTypeRules
|
|
569
643
|
);
|
|
570
644
|
|
|
571
645
|
// 7. Upsert document - EXPLICITLY clear error fields on success
|
|
@@ -588,6 +662,7 @@ export class SyncService {
|
|
|
588
662
|
author: extractedMetadata.author,
|
|
589
663
|
frontmatterDate: extractedMetadata.frontmatterDate,
|
|
590
664
|
dateFields: extractedMetadata.dateFields,
|
|
665
|
+
contentTypeRulesFingerprint,
|
|
591
666
|
// Clear error fields on success (requires store to handle undefined → null)
|
|
592
667
|
lastErrorCode: undefined,
|
|
593
668
|
lastErrorMessage: undefined,
|
|
@@ -711,6 +786,8 @@ export class SyncService {
|
|
|
711
786
|
status,
|
|
712
787
|
docid,
|
|
713
788
|
mirrorHash: artifact.mirrorHash,
|
|
789
|
+
contentType: extractedMetadata.contentType,
|
|
790
|
+
contentTypeSource: extractedMetadata.contentTypeSource,
|
|
714
791
|
};
|
|
715
792
|
} catch (error) {
|
|
716
793
|
const message = error instanceof Error ? error.message : "Unknown error";
|
|
@@ -828,6 +905,13 @@ export class SyncService {
|
|
|
828
905
|
options: SyncOptions = {}
|
|
829
906
|
): Promise<CollectionSyncResult> {
|
|
830
907
|
const startTime = Date.now();
|
|
908
|
+
const syncOptions: SyncOptions = {
|
|
909
|
+
...options,
|
|
910
|
+
contentTypeRules: options.contentTypeRules ?? [],
|
|
911
|
+
contentTypeRulesFingerprint:
|
|
912
|
+
options.contentTypeRulesFingerprint ??
|
|
913
|
+
fingerprintContentTypeRules(options.contentTypeRules ?? []),
|
|
914
|
+
};
|
|
831
915
|
const errors: Array<{ relPath: string; code: string; message: string }> =
|
|
832
916
|
[];
|
|
833
917
|
|
|
@@ -892,6 +976,7 @@ export class SyncService {
|
|
|
892
976
|
let unchanged = 0;
|
|
893
977
|
let errored = 0;
|
|
894
978
|
let dynamicSkipped = 0;
|
|
979
|
+
const fileResults: FileSyncResult[] = [];
|
|
895
980
|
|
|
896
981
|
if (concurrency === 1) {
|
|
897
982
|
// Sequential processing with batched transactions (Windows perf)
|
|
@@ -905,8 +990,9 @@ export class SyncService {
|
|
|
905
990
|
collection,
|
|
906
991
|
entry,
|
|
907
992
|
store,
|
|
908
|
-
|
|
993
|
+
syncOptions
|
|
909
994
|
);
|
|
995
|
+
fileResults.push(result);
|
|
910
996
|
switch (result.status) {
|
|
911
997
|
case "added":
|
|
912
998
|
added += 1;
|
|
@@ -970,8 +1056,9 @@ export class SyncService {
|
|
|
970
1056
|
collection,
|
|
971
1057
|
entry,
|
|
972
1058
|
store,
|
|
973
|
-
|
|
1059
|
+
syncOptions
|
|
974
1060
|
);
|
|
1061
|
+
fileResults.push(result);
|
|
975
1062
|
results.push(result);
|
|
976
1063
|
} finally {
|
|
977
1064
|
semaphore.release();
|
|
@@ -1044,6 +1131,7 @@ export class SyncService {
|
|
|
1044
1131
|
filesSkipped: skipped.length + dynamicSkipped,
|
|
1045
1132
|
filesMarkedInactive: markedInactive,
|
|
1046
1133
|
durationMs: Date.now() - startTime,
|
|
1134
|
+
files: fileResults,
|
|
1047
1135
|
errors,
|
|
1048
1136
|
};
|
|
1049
1137
|
}
|
package/src/ingestion/types.ts
CHANGED
|
@@ -5,6 +5,7 @@
|
|
|
5
5
|
* @module src/ingestion/types
|
|
6
6
|
*/
|
|
7
7
|
|
|
8
|
+
import type { NormalizedContentTypeRule } from "../config";
|
|
8
9
|
import type { Collection } from "../config/types";
|
|
9
10
|
|
|
10
11
|
// ─────────────────────────────────────────────────────────────────────────────
|
|
@@ -132,8 +133,18 @@ export interface SyncOptions {
|
|
|
132
133
|
* SQLite operations are serialized regardless of this setting.
|
|
133
134
|
*/
|
|
134
135
|
concurrency?: number;
|
|
136
|
+
/** Normalized content type rules from config.contentTypes. */
|
|
137
|
+
contentTypeRules?: NormalizedContentTypeRule[];
|
|
138
|
+
/** Stable hash of the normalized content type rules, used for re-derivation. */
|
|
139
|
+
contentTypeRulesFingerprint?: string;
|
|
135
140
|
}
|
|
136
141
|
|
|
142
|
+
export type ContentTypeSource =
|
|
143
|
+
| "frontmatter-type"
|
|
144
|
+
| "prefix"
|
|
145
|
+
| "path-ext"
|
|
146
|
+
| "fallback";
|
|
147
|
+
|
|
137
148
|
/** Per-file sync status */
|
|
138
149
|
export type FileSyncStatus =
|
|
139
150
|
| "added"
|
|
@@ -148,6 +159,8 @@ export interface FileSyncResult {
|
|
|
148
159
|
status: FileSyncStatus;
|
|
149
160
|
docid?: string;
|
|
150
161
|
mirrorHash?: string;
|
|
162
|
+
contentType?: string;
|
|
163
|
+
contentTypeSource?: ContentTypeSource;
|
|
151
164
|
errorCode?: string;
|
|
152
165
|
errorMessage?: string;
|
|
153
166
|
}
|
|
@@ -163,6 +176,7 @@ export interface CollectionSyncResult {
|
|
|
163
176
|
filesSkipped: number;
|
|
164
177
|
filesMarkedInactive: number;
|
|
165
178
|
durationMs: number;
|
|
179
|
+
files?: FileSyncResult[];
|
|
166
180
|
errors: Array<{
|
|
167
181
|
relPath: string;
|
|
168
182
|
code: string;
|
|
@@ -18,7 +18,7 @@ import {
|
|
|
18
18
|
normalizeCollectionName,
|
|
19
19
|
validateCollectionRoot,
|
|
20
20
|
} from "../../core/validation";
|
|
21
|
-
import { defaultSyncService } from "../../ingestion";
|
|
21
|
+
import { defaultSyncService, withContentTypeRules } from "../../ingestion";
|
|
22
22
|
import { runTool, type ToolResult } from "./index";
|
|
23
23
|
|
|
24
24
|
interface AddCollectionInput {
|
|
@@ -147,10 +147,13 @@ export function handleAddCollection(
|
|
|
147
147
|
const result = await defaultSyncService.syncCollection(
|
|
148
148
|
collection,
|
|
149
149
|
ctx.store,
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
150
|
+
withContentTypeRules(
|
|
151
|
+
{
|
|
152
|
+
gitPull: args.gitPull ?? false,
|
|
153
|
+
runUpdateCmd: false,
|
|
154
|
+
},
|
|
155
|
+
ctx.config
|
|
156
|
+
)
|
|
154
157
|
);
|
|
155
158
|
|
|
156
159
|
return {
|