@gobing-ai/knowledge-kit 0.0.7 → 0.0.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +21604 -11844
- package/package.json +4 -1
- package/plugins/generations/content-gen/package.json +2 -1
- package/plugins/generations/content-gen/src/index.ts +9 -8
- package/plugins/generations/content-gen/src/storm.ts +12 -4
- package/plugins/generations/dailynews-gen/package.json +17 -0
- package/plugins/generations/dailynews-gen/plugin.json +7 -0
- package/plugins/generations/dailynews-gen/src/index.ts +111 -0
- package/plugins/generations/dailynews-gen/src/script-builder.ts +121 -0
- package/plugins/generations/dailynews-gen/tsconfig.json +4 -0
- package/plugins/generations/voice-gen/package.json +17 -0
- package/plugins/generations/voice-gen/plugin.json +6 -0
- package/plugins/generations/voice-gen/src/concat.ts +218 -0
- package/plugins/generations/voice-gen/src/index.ts +228 -0
- package/plugins/generations/voice-gen/src/mp3.ts +65 -0
- package/plugins/generations/voice-gen/src/voicebox-client.ts +223 -0
- package/plugins/generations/voice-gen/src/voicescript.ts +365 -0
- package/plugins/generations/voice-gen/tsconfig.json +8 -0
- package/plugins/ingestions/aihot-ingest/package.json +17 -0
- package/plugins/ingestions/aihot-ingest/plugin.json +7 -0
- package/plugins/ingestions/aihot-ingest/src/client.ts +185 -0
- package/plugins/ingestions/aihot-ingest/src/index.ts +137 -0
- package/plugins/ingestions/aihot-ingest/src/mapper.ts +42 -0
- package/plugins/ingestions/aihot-ingest/tsconfig.json +4 -0
- package/plugins/ingestions/karakeep-local/package.json +17 -0
- package/plugins/ingestions/karakeep-local/src/index.ts +31 -26
- package/plugins/ingestions/karakeep-local/tsconfig.json +4 -0
- package/plugins/ingestions/web-search/package.json +4 -1
- package/plugins/ingestions/web-search/src/index.ts +139 -16
- package/plugins/kk/README.md +9 -3
- package/plugins/kk/commands/workflow-run.md +100 -28
- package/plugins/kk/config.example.yaml +34 -0
- package/plugins/kk/scripts/render-md.ts +8 -3
- package/plugins/kk/skills/{judge → content-judge}/SKILL.md +13 -14
- package/plugins/kk/skills/{judge → content-judge}/references/workflow-integration.md +13 -11
- package/plugins/kk/skills/itc-generating/SKILL.md +147 -0
- package/plugins/kk/skills/itc-generating/references/generic-craft.md +80 -0
- package/plugins/kk/skills/itc-generating/references/platform-english.md +72 -0
- package/plugins/kk/skills/itc-generating/references/platform-wechat.md +60 -0
- package/plugins/kk/skills/itc-generating/references/skill-authoring.md +62 -0
- package/plugins/kk/skills/storm-research/SKILL.md +10 -3
- package/plugins/kk/workflows/judge-gated-publish-example.yaml +101 -0
- package/plugins/kk/workflows/kk-daily-ai-voice.yaml +144 -0
- package/plugins/kk/workflows/kk-ingest-generate-publish.yaml +72 -0
- package/plugins/kk/workflows/kk-itc.yaml +285 -0
- package/plugins/kk/workflows/kk-solo-podcast.yaml +374 -0
- package/plugins/kk/workflows/validate-voicescript.ts +226 -0
- package/plugins/publishings/emdash-pub/package.json +17 -0
- package/plugins/publishings/emdash-pub/plugin.json +7 -0
- package/plugins/publishings/emdash-pub/src/index.ts +450 -0
- package/plugins/publishings/emdash-pub/tsconfig.json +4 -0
- package/plugins/publishings/qiita-pub/package.json +2 -1
- package/plugins/publishings/qiita-pub/src/index.ts +9 -9
- package/plugins/publishings/surfdash-pub/package.json +2 -1
- package/plugins/publishings/surfdash-pub/src/index.ts +17 -12
- package/plugins/publishings/zenn-pub/package.json +2 -1
- package/plugins/publishings/zenn-pub/src/index.ts +11 -11
- package/plugins/kk/agents/judge-compliance.md +0 -37
- package/plugins/kk/agents/judge-tech.md +0 -35
- package/plugins/kk/agents/judge-tone.md +0 -37
- /package/plugins/kk/skills/{judge → content-judge}/references/rubrics.md +0 -0
|
@@ -0,0 +1,137 @@
|
|
|
1
|
+
import { dirname } from 'node:path';
|
|
2
|
+
import { parseArgs } from 'node:util';
|
|
3
|
+
import { type Doc, DocListSchema } from '@gobing-ai/kk-core';
|
|
4
|
+
import { atomicWriteJson, createNodeFileSystem, readJsonFile } from '@gobing-ai/ts-runtime';
|
|
5
|
+
import { echoError } from '@gobing-ai/ts-utils';
|
|
6
|
+
import { z } from 'zod';
|
|
7
|
+
import { type AihotDeps, createAihotClient } from './client';
|
|
8
|
+
import { mapAihotItemsToDocs } from './mapper';
|
|
9
|
+
|
|
10
|
+
export * from './client';
|
|
11
|
+
export * from './mapper';
|
|
12
|
+
|
|
13
|
+
export interface IngestionOptions {
|
|
14
|
+
out: string;
|
|
15
|
+
limit?: number;
|
|
16
|
+
cursor?: string;
|
|
17
|
+
stateFile?: string;
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
export interface IngestionResult {
|
|
21
|
+
docs: Doc[];
|
|
22
|
+
cursor: string;
|
|
23
|
+
asOf?: string;
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
const IngestionInputSchema = z.object({
|
|
27
|
+
limit: z.number().int().min(1).max(100).optional(),
|
|
28
|
+
cursor: z.string().trim().min(1).optional(),
|
|
29
|
+
stateFile: z.string().trim().min(1).optional(),
|
|
30
|
+
});
|
|
31
|
+
|
|
32
|
+
const CursorStateSchema = z.object({ cursor: z.string().trim().min(1) });
|
|
33
|
+
|
|
34
|
+
function parseLimit(value: string): number {
|
|
35
|
+
return z.coerce.number().int().min(1).max(100).parse(value);
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
export async function processIngestionIO(
|
|
39
|
+
options: IngestionOptions,
|
|
40
|
+
depsOverride?: Partial<AihotDeps>,
|
|
41
|
+
): Promise<IngestionResult> {
|
|
42
|
+
const fs = createNodeFileSystem();
|
|
43
|
+
const client = createAihotClient(depsOverride);
|
|
44
|
+
|
|
45
|
+
let effectiveCursor = options.cursor?.trim();
|
|
46
|
+
if (!effectiveCursor && options.stateFile && (await fs.exists(options.stateFile))) {
|
|
47
|
+
effectiveCursor = CursorStateSchema.parse(await readJsonFile(options.stateFile, fs)).cursor;
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
let items: import('./client').AihotItem[];
|
|
51
|
+
let cursor: string;
|
|
52
|
+
let asOf: string | undefined;
|
|
53
|
+
|
|
54
|
+
if (effectiveCursor) {
|
|
55
|
+
const changes = await client.fetchChanges({
|
|
56
|
+
cursor: effectiveCursor,
|
|
57
|
+
limit: options.limit,
|
|
58
|
+
});
|
|
59
|
+
items = changes.changes.flatMap((change) => (change.op === 'upsert' ? [change.item] : []));
|
|
60
|
+
cursor = changes.cursor;
|
|
61
|
+
} else {
|
|
62
|
+
const snapshot = await client.fetchSnapshot({
|
|
63
|
+
limit: options.limit,
|
|
64
|
+
});
|
|
65
|
+
items = snapshot.items;
|
|
66
|
+
cursor = snapshot.cursor;
|
|
67
|
+
asOf = snapshot.asOf;
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
const docs = DocListSchema.parse(mapAihotItemsToDocs(items));
|
|
71
|
+
|
|
72
|
+
const outDir = dirname(options.out);
|
|
73
|
+
if (outDir && outDir !== '.') {
|
|
74
|
+
await fs.ensureDir(outDir);
|
|
75
|
+
}
|
|
76
|
+
await atomicWriteJson(options.out, docs, fs);
|
|
77
|
+
|
|
78
|
+
if (options.stateFile) {
|
|
79
|
+
const stateDir = dirname(options.stateFile);
|
|
80
|
+
if (stateDir && stateDir !== '.') {
|
|
81
|
+
await fs.ensureDir(stateDir);
|
|
82
|
+
}
|
|
83
|
+
const statePayload = {
|
|
84
|
+
cursor,
|
|
85
|
+
asOf,
|
|
86
|
+
lastSyncAt: new Date().toISOString(),
|
|
87
|
+
itemCount: docs.length,
|
|
88
|
+
};
|
|
89
|
+
await atomicWriteJson(options.stateFile, statePayload, fs);
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
return { docs, cursor, asOf };
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
export async function main(depsOverride?: Partial<AihotDeps>): Promise<number> {
|
|
96
|
+
let values: { in?: string; out?: string; limit?: string; cursor?: string; 'state-file'?: string };
|
|
97
|
+
try {
|
|
98
|
+
({ values } = parseArgs({
|
|
99
|
+
options: {
|
|
100
|
+
in: { type: 'string' },
|
|
101
|
+
out: { type: 'string' },
|
|
102
|
+
limit: { type: 'string' },
|
|
103
|
+
cursor: { type: 'string' },
|
|
104
|
+
'state-file': { type: 'string' },
|
|
105
|
+
},
|
|
106
|
+
}));
|
|
107
|
+
} catch (err: unknown) {
|
|
108
|
+
echoError(`aihot-ingest failed: ${err instanceof Error ? err.message : String(err)}`);
|
|
109
|
+
return 1;
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
if (!values.out) {
|
|
113
|
+
echoError('aihot-ingest failed: Missing required argument: --out');
|
|
114
|
+
return 1;
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
try {
|
|
118
|
+
const input = values.in ? IngestionInputSchema.parse(await readJsonFile(values.in)) : {};
|
|
119
|
+
await processIngestionIO(
|
|
120
|
+
{
|
|
121
|
+
out: values.out,
|
|
122
|
+
limit: values.limit ? parseLimit(values.limit) : (input.limit ?? 10),
|
|
123
|
+
cursor: values.cursor ?? input.cursor,
|
|
124
|
+
stateFile: values['state-file'] ?? input.stateFile,
|
|
125
|
+
},
|
|
126
|
+
depsOverride,
|
|
127
|
+
);
|
|
128
|
+
return 0;
|
|
129
|
+
} catch (err: unknown) {
|
|
130
|
+
echoError(`aihot-ingest failed: ${err instanceof Error ? err.message : String(err)}`);
|
|
131
|
+
return 1;
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
if (import.meta.main) {
|
|
136
|
+
process.exit(await main());
|
|
137
|
+
}
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
import { createHash } from 'node:crypto';
|
|
2
|
+
import type { Doc } from '@gobing-ai/kk-core';
|
|
3
|
+
import type { AihotItem } from './client';
|
|
4
|
+
|
|
5
|
+
export function computeDocId(seed: string): string {
|
|
6
|
+
return createHash('sha256').update(seed).digest('hex').slice(0, 16);
|
|
7
|
+
}
|
|
8
|
+
|
|
9
|
+
export function mapAihotItemToDoc(item: AihotItem): Doc {
|
|
10
|
+
const rawId = item.id.trim();
|
|
11
|
+
const canonicalSource = item.links.original || item.links.aihot || item.title || 'untitled';
|
|
12
|
+
const id = rawId.length > 0 ? rawId : computeDocId(canonicalSource);
|
|
13
|
+
|
|
14
|
+
const sourceUri = item.links?.original || item.links?.aihot || `https://aihot.virxact.com/items/${id}`;
|
|
15
|
+
const title = item.title.trim() || 'Untitled AI News';
|
|
16
|
+
const body = `${item.summary?.trim() || item.title.trim()}\n`;
|
|
17
|
+
|
|
18
|
+
const metadata: Record<string, unknown> = {
|
|
19
|
+
sourceName: item.source.name,
|
|
20
|
+
publishedAt: item.publishedAt,
|
|
21
|
+
discoveredAt: item.discoveredAt,
|
|
22
|
+
category: item.category ?? undefined,
|
|
23
|
+
score: item.score ?? undefined,
|
|
24
|
+
aihotUrl: item.links.aihot,
|
|
25
|
+
originalUrl: item.links.original,
|
|
26
|
+
attribution: item.attribution,
|
|
27
|
+
reason: item.reason ?? undefined,
|
|
28
|
+
};
|
|
29
|
+
|
|
30
|
+
return {
|
|
31
|
+
id,
|
|
32
|
+
title,
|
|
33
|
+
sourceUri,
|
|
34
|
+
body,
|
|
35
|
+
mediaType: 'text/markdown',
|
|
36
|
+
metadata,
|
|
37
|
+
};
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
export function mapAihotItemsToDocs(items: AihotItem[]): Doc[] {
|
|
41
|
+
return items.map(mapAihotItemToDoc);
|
|
42
|
+
}
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "@gobing-ai/karakeep-local",
|
|
3
|
+
"type": "module",
|
|
4
|
+
"private": true,
|
|
5
|
+
"scripts": {
|
|
6
|
+
"typecheck": "tsc --noEmit"
|
|
7
|
+
},
|
|
8
|
+
"dependencies": {
|
|
9
|
+
"@gobing-ai/kk-core": "workspace:*",
|
|
10
|
+
"@gobing-ai/ts-infra": "catalog:",
|
|
11
|
+
"@gobing-ai/ts-runtime": "catalog:",
|
|
12
|
+
"@gobing-ai/ts-utils": "catalog:"
|
|
13
|
+
},
|
|
14
|
+
"devDependencies": {
|
|
15
|
+
"@types/bun": "1.3.14"
|
|
16
|
+
}
|
|
17
|
+
}
|
|
@@ -1,8 +1,12 @@
|
|
|
1
1
|
import { createHash } from 'node:crypto';
|
|
2
|
-
import {
|
|
3
|
-
import { dirname, join, relative } from 'node:path';
|
|
2
|
+
import { dirname, relative } from 'node:path';
|
|
4
3
|
import { parseArgs } from 'node:util';
|
|
5
4
|
import type { Doc } from '@gobing-ai/kk-core';
|
|
5
|
+
import { getLogger, initializeLogger } from '@gobing-ai/ts-infra';
|
|
6
|
+
import { createNodeFileSystem, walkDir } from '@gobing-ai/ts-runtime';
|
|
7
|
+
import { echoError } from '@gobing-ai/ts-utils';
|
|
8
|
+
|
|
9
|
+
const log = getLogger('kk.ingestion.karakeep-local');
|
|
6
10
|
|
|
7
11
|
interface ParsedMeta {
|
|
8
12
|
title?: string;
|
|
@@ -17,27 +21,26 @@ function computeHash(str: string): string {
|
|
|
17
21
|
}
|
|
18
22
|
|
|
19
23
|
async function collectFiles(dir: string): Promise<string[]> {
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
const fullPath = join(dir, entry);
|
|
24
|
-
const st = await stat(fullPath);
|
|
25
|
-
if (st.isFile()) {
|
|
26
|
-
files.push(fullPath);
|
|
27
|
-
}
|
|
28
|
-
}
|
|
29
|
-
return files.sort();
|
|
24
|
+
// walkDir is cycle-safe and root-confined: a directory symlink pointing back at
|
|
25
|
+
// an ancestor is walked at most once, and one escaping `dir` is skipped.
|
|
26
|
+
return (await walkDir(dir, createNodeFileSystem())).sort();
|
|
30
27
|
}
|
|
31
28
|
|
|
32
29
|
export async function processKarakeepFolder(inDir: string): Promise<Doc[]> {
|
|
30
|
+
const fs = createNodeFileSystem();
|
|
33
31
|
try {
|
|
34
|
-
|
|
32
|
+
// The FileSystem seam returns null for a missing path where node's stat throws;
|
|
33
|
+
// both cases must still surface as "Inaccessible source path" below.
|
|
34
|
+
const inStat = await fs.stat(inDir);
|
|
35
|
+
if (inStat === null) {
|
|
36
|
+
throw new Error(`Source path does not exist: ${inDir}`);
|
|
37
|
+
}
|
|
35
38
|
if (!inStat.isDirectory()) {
|
|
36
39
|
throw new Error(`Source path is not a directory: ${inDir}`);
|
|
37
40
|
}
|
|
38
41
|
} catch (err: unknown) {
|
|
39
42
|
const msg = `Inaccessible source path: ${inDir}`;
|
|
40
|
-
|
|
43
|
+
echoError(`Error: ${msg}`);
|
|
41
44
|
throw new Error(msg, { cause: err });
|
|
42
45
|
}
|
|
43
46
|
|
|
@@ -62,14 +65,14 @@ export async function processKarakeepFolder(inDir: string): Promise<Doc[]> {
|
|
|
62
65
|
|
|
63
66
|
if (contentMd) {
|
|
64
67
|
processedFiles.add(contentMd);
|
|
65
|
-
const mdText = await readFile(contentMd
|
|
68
|
+
const mdText = await fs.readFile(contentMd);
|
|
66
69
|
let metaObj: ParsedMeta = {};
|
|
67
70
|
if (metaJson) {
|
|
68
71
|
processedFiles.add(metaJson);
|
|
69
72
|
try {
|
|
70
|
-
metaObj = JSON.parse(await readFile(metaJson
|
|
73
|
+
metaObj = JSON.parse(await fs.readFile(metaJson)) as ParsedMeta;
|
|
71
74
|
} catch (err: unknown) {
|
|
72
|
-
|
|
75
|
+
log.warn(`Warning: corrupt/unparsable meta file at ${metaJson}`, { error: err });
|
|
73
76
|
}
|
|
74
77
|
}
|
|
75
78
|
|
|
@@ -99,7 +102,7 @@ export async function processKarakeepFolder(inDir: string): Promise<Doc[]> {
|
|
|
99
102
|
|
|
100
103
|
if (f.endsWith('.json')) {
|
|
101
104
|
try {
|
|
102
|
-
const jsonText = await readFile(f
|
|
105
|
+
const jsonText = await fs.readFile(f);
|
|
103
106
|
const jsonObj = JSON.parse(jsonText) as ParsedMeta;
|
|
104
107
|
const { title, sourceUri, mediaType, body, ...extraMeta } = jsonObj;
|
|
105
108
|
if (typeof body === 'string') {
|
|
@@ -112,13 +115,13 @@ export async function processKarakeepFolder(inDir: string): Promise<Doc[]> {
|
|
|
112
115
|
metadata: Object.keys(extraMeta).length > 0 ? extraMeta : undefined,
|
|
113
116
|
});
|
|
114
117
|
} else {
|
|
115
|
-
|
|
118
|
+
log.warn(`Warning: JSON entry missing string body at ${f}`);
|
|
116
119
|
}
|
|
117
120
|
} catch (err: unknown) {
|
|
118
|
-
|
|
121
|
+
log.warn(`Warning: corrupt/unparsable JSON file at ${f}`, { error: err });
|
|
119
122
|
}
|
|
120
123
|
} else if (f.endsWith('.md')) {
|
|
121
|
-
const mdText = await readFile(f
|
|
124
|
+
const mdText = await fs.readFile(f);
|
|
122
125
|
docs.push({
|
|
123
126
|
id,
|
|
124
127
|
body: mdText,
|
|
@@ -140,31 +143,33 @@ export async function main() {
|
|
|
140
143
|
});
|
|
141
144
|
|
|
142
145
|
if (!values.in || !values.out) {
|
|
143
|
-
|
|
146
|
+
echoError('Error: Missing required arguments --in or --out');
|
|
144
147
|
process.exit(1);
|
|
145
148
|
}
|
|
146
149
|
|
|
150
|
+
const fs = createNodeFileSystem();
|
|
147
151
|
try {
|
|
148
152
|
const candidateFiles = await collectFiles(values.in);
|
|
149
153
|
const docs = await processKarakeepFolder(values.in);
|
|
150
154
|
|
|
151
155
|
if (docs.length === 0 && candidateFiles.length > 0) {
|
|
152
|
-
|
|
156
|
+
echoError(`Fatal: source yielded zero ingestible documents at ${values.in}`);
|
|
153
157
|
process.exit(1);
|
|
154
158
|
}
|
|
155
159
|
|
|
156
160
|
const outDir = dirname(values.out);
|
|
157
161
|
if (outDir && outDir !== '.') {
|
|
158
|
-
await
|
|
162
|
+
await fs.ensureDir(outDir);
|
|
159
163
|
}
|
|
160
164
|
|
|
161
|
-
await writeFile(values.out, JSON.stringify(docs, null, 2)
|
|
165
|
+
await fs.writeFile(values.out, JSON.stringify(docs, null, 2));
|
|
162
166
|
} catch (err: unknown) {
|
|
163
|
-
|
|
167
|
+
echoError(`Fatal error during karakeep-local ingestion: ${err instanceof Error ? err.message : String(err)}`);
|
|
164
168
|
process.exit(1);
|
|
165
169
|
}
|
|
166
170
|
}
|
|
167
171
|
|
|
168
172
|
if (import.meta.main) {
|
|
173
|
+
await initializeLogger({ console: process.env.NODE_ENV !== 'test', json: true, level: 'warn' });
|
|
169
174
|
main();
|
|
170
175
|
}
|
|
@@ -6,7 +6,10 @@
|
|
|
6
6
|
"typecheck": "tsc --noEmit"
|
|
7
7
|
},
|
|
8
8
|
"dependencies": {
|
|
9
|
-
"@gobing-ai/kk-core": "workspace:*"
|
|
9
|
+
"@gobing-ai/kk-core": "workspace:*",
|
|
10
|
+
"@gobing-ai/ts-infra": "catalog:",
|
|
11
|
+
"@gobing-ai/ts-runtime": "catalog:",
|
|
12
|
+
"@gobing-ai/ts-utils": "catalog:"
|
|
10
13
|
},
|
|
11
14
|
"devDependencies": {
|
|
12
15
|
"@types/bun": "1.3.14"
|
|
@@ -1,8 +1,12 @@
|
|
|
1
1
|
import { createHash } from 'node:crypto';
|
|
2
|
-
import {
|
|
3
|
-
import { dirname } from 'node:path';
|
|
2
|
+
import { dirname, join } from 'node:path';
|
|
4
3
|
import { parseArgs } from 'node:util';
|
|
5
4
|
import { type Doc, DocListSchema } from '@gobing-ai/kk-core';
|
|
5
|
+
import { getLogger, initializeLogger } from '@gobing-ai/ts-infra';
|
|
6
|
+
import { createNodeFileSystem } from '@gobing-ai/ts-runtime';
|
|
7
|
+
import { echoError } from '@gobing-ai/ts-utils';
|
|
8
|
+
|
|
9
|
+
const log = getLogger('kk.ingestion.web-search');
|
|
6
10
|
|
|
7
11
|
/** Validated `--in` input per the 0054 contract (`topic` required; `maxResults`/`fixture` optional). */
|
|
8
12
|
interface InInput {
|
|
@@ -49,13 +53,49 @@ export function computeId(uri: string): string {
|
|
|
49
53
|
/** Fetch signature the live path accepts, so tests can inject a fake (CI never hits the network). */
|
|
50
54
|
export type FetchLike = (input: string, init?: RequestInit) => Promise<Response>;
|
|
51
55
|
|
|
52
|
-
/**
|
|
56
|
+
/** markitdown normalizer — HTML in, markdown out (ADR-013 D3). Throws on non-zero exit or
|
|
57
|
+
* empty output; callers treat a throw as unusable content (per-URL fallback, D5). */
|
|
58
|
+
export type MarkitdownRunner = (html: string) => Promise<string>;
|
|
59
|
+
|
|
60
|
+
/** Live-path dependencies — fetch, env, clock, scrape, and markitdown are all injectable for tests. */
|
|
53
61
|
export interface SearchDeps {
|
|
54
62
|
fetch: FetchLike;
|
|
55
63
|
env: Record<string, string | undefined>;
|
|
56
64
|
now: () => Date;
|
|
65
|
+
/** Scrape transport; defaults to `deps.fetch` so tests can route both Firecrawl calls separately. */
|
|
66
|
+
scrape?: FetchLike;
|
|
67
|
+
/** markitdown runner; when absent in live mode, `main` probes PATH and fails loud (R6). */
|
|
68
|
+
markitdown?: MarkitdownRunner;
|
|
57
69
|
}
|
|
58
70
|
|
|
71
|
+
/** Real markitdown runner: `markitdown -x .html`, HTML on stdin, markdown on stdout. The
|
|
72
|
+
* `-x .html` extension hint is required — bare stdin passes HTML through unconverted.
|
|
73
|
+
* Spawns the absolute path resolved via Bun.which (with an explicit PATH option — plain
|
|
74
|
+
* argv[0] lookup can resolve against a startup PATH snapshot, missing runtime edits). */
|
|
75
|
+
export const realMarkitdown: MarkitdownRunner = async (html: string): Promise<string> => {
|
|
76
|
+
const bin = Bun.which('markitdown', { PATH: process.env.PATH });
|
|
77
|
+
if (!bin) {
|
|
78
|
+
throw new Error('markitdown is not on PATH');
|
|
79
|
+
}
|
|
80
|
+
const proc = Bun.spawn([bin, '-x', '.html'], { stdin: 'pipe', stdout: 'pipe', stderr: 'pipe' });
|
|
81
|
+
proc.stdin.write(html);
|
|
82
|
+
await proc.stdin.end();
|
|
83
|
+
const [stdout, stderrText, exitCode] = await Promise.all([
|
|
84
|
+
new Response(proc.stdout).text(),
|
|
85
|
+
new Response(proc.stderr).text(),
|
|
86
|
+
proc.exited,
|
|
87
|
+
]);
|
|
88
|
+
if (exitCode !== 0) {
|
|
89
|
+
const detail = stderrText.trim();
|
|
90
|
+
throw new Error(`markitdown exited ${exitCode}${detail ? `: ${detail}` : ''}`);
|
|
91
|
+
}
|
|
92
|
+
const markdown = stdout.trim();
|
|
93
|
+
if (markdown.length === 0) {
|
|
94
|
+
throw new Error('markitdown produced empty output');
|
|
95
|
+
}
|
|
96
|
+
return markdown;
|
|
97
|
+
};
|
|
98
|
+
|
|
59
99
|
/** One Firecrawl `data.web[]` item — the subset of fields 0054 Block 3/4 consume. */
|
|
60
100
|
interface FirecrawlWebItem {
|
|
61
101
|
url?: unknown;
|
|
@@ -70,6 +110,12 @@ interface FirecrawlSearchResponse {
|
|
|
70
110
|
data?: { web?: FirecrawlWebItem[] | null } | null;
|
|
71
111
|
}
|
|
72
112
|
|
|
113
|
+
/** One Firecrawl `/v2/scrape` body — the subset the cache pass consumes. */
|
|
114
|
+
interface FirecrawlScrapeResponse {
|
|
115
|
+
success?: unknown;
|
|
116
|
+
data?: { html?: unknown; markdown?: unknown } | null;
|
|
117
|
+
}
|
|
118
|
+
|
|
73
119
|
/**
|
|
74
120
|
* Live Firecrawl search → Doc[] (0054 Blocks 2–5). Throws an Error whose message is the
|
|
75
121
|
* fail-loud diagnostic; `main` prefixes it with `web-search failed: ` and returns 1.
|
|
@@ -152,7 +198,7 @@ export async function searchFirecrawl(input: InInput, deps: SearchDeps): Promise
|
|
|
152
198
|
if (!url || body.length === 0) {
|
|
153
199
|
// Row 7 — partial item failure: skip + stderr warn, continue.
|
|
154
200
|
skipped += 1;
|
|
155
|
-
|
|
201
|
+
log.warn(`web-search: skipping result without usable markdown (url=${url ?? 'missing'})`);
|
|
156
202
|
continue;
|
|
157
203
|
}
|
|
158
204
|
docs.push({
|
|
@@ -195,6 +241,72 @@ function extractError(bodyText: string): string {
|
|
|
195
241
|
return '';
|
|
196
242
|
}
|
|
197
243
|
|
|
244
|
+
/** Scrape + normalize + cache pass over the search results (ADR-013 D1–D6). Sequential
|
|
245
|
+
* per Doc — deliberate, no parallel fan-out (rate-limit friendly, deterministic). Cache hit
|
|
246
|
+
* (`references/<doc-id>.md` exists) reuses the file with no new /v2/scrape call (R3/R4);
|
|
247
|
+
* otherwise scrape → normalize (html via markitdown, markdown as-is) → write cache → body
|
|
248
|
+
* swap. Per-URL failure or unusable content warns and keeps the snippet body (R5/R11) — never
|
|
249
|
+
* fatal. Returns a new Doc[]; input order and non-body fields are untouched. */
|
|
250
|
+
export async function scrapeAndCacheDocs(docs: Doc[], deps: SearchDeps, outPath: string): Promise<Doc[]> {
|
|
251
|
+
const fs = createNodeFileSystem();
|
|
252
|
+
const cacheDir = join(dirname(outPath), 'references');
|
|
253
|
+
const scrapeFetch = deps.scrape ?? deps.fetch;
|
|
254
|
+
const out: Doc[] = [];
|
|
255
|
+
for (const doc of docs) {
|
|
256
|
+
const uri = doc.sourceUri;
|
|
257
|
+
if (!uri) {
|
|
258
|
+
out.push(doc);
|
|
259
|
+
continue;
|
|
260
|
+
}
|
|
261
|
+
const cacheFile = join(cacheDir, `${doc.id}.md`);
|
|
262
|
+
try {
|
|
263
|
+
const cached = await fs.readFile(cacheFile);
|
|
264
|
+
if (cached.trim().length > 0) {
|
|
265
|
+
out.push({ ...doc, body: cached });
|
|
266
|
+
continue;
|
|
267
|
+
}
|
|
268
|
+
} catch {
|
|
269
|
+
// Cache miss — fall through to the live scrape.
|
|
270
|
+
}
|
|
271
|
+
try {
|
|
272
|
+
const key = deps.env.FIRECRAWL_API_KEY;
|
|
273
|
+
if (!key) throw new Error('FIRECRAWL_API_KEY is unset');
|
|
274
|
+
const res = await scrapeFetch('https://api.firecrawl.dev/v2/scrape', {
|
|
275
|
+
method: 'POST',
|
|
276
|
+
headers: {
|
|
277
|
+
'Content-Type': 'application/json',
|
|
278
|
+
Authorization: `Bearer ${key}`,
|
|
279
|
+
},
|
|
280
|
+
body: JSON.stringify({ url: uri, formats: ['html', 'markdown'] }),
|
|
281
|
+
});
|
|
282
|
+
if (!res.ok || res.status !== 200) {
|
|
283
|
+
throw new Error(`Firecrawl HTTP ${res.status}`);
|
|
284
|
+
}
|
|
285
|
+
const parsed = JSON.parse(await res.text()) as FirecrawlScrapeResponse;
|
|
286
|
+
if (parsed.success !== true || parsed.data === null || parsed.data === undefined) {
|
|
287
|
+
throw new Error('Firecrawl malformed scrape response');
|
|
288
|
+
}
|
|
289
|
+
const html = typeof parsed.data.html === 'string' ? parsed.data.html : undefined;
|
|
290
|
+
const markdown = typeof parsed.data.markdown === 'string' ? parsed.data.markdown : undefined;
|
|
291
|
+
let normalized: string;
|
|
292
|
+
if (html && html.trim().length > 0) {
|
|
293
|
+
normalized = await (deps.markitdown ?? realMarkitdown)(html);
|
|
294
|
+
} else if (markdown && markdown.trim().length > 0) {
|
|
295
|
+
normalized = markdown.trim();
|
|
296
|
+
} else {
|
|
297
|
+
throw new Error('scrape returned no usable content');
|
|
298
|
+
}
|
|
299
|
+
await fs.ensureDir(cacheDir);
|
|
300
|
+
await fs.writeFile(cacheFile, normalized);
|
|
301
|
+
out.push({ ...doc, body: normalized });
|
|
302
|
+
} catch (err: unknown) {
|
|
303
|
+
log.warn(`web-search: scrape failed (url=${uri}): ${err instanceof Error ? err.message : String(err)}`);
|
|
304
|
+
out.push(doc);
|
|
305
|
+
}
|
|
306
|
+
}
|
|
307
|
+
return out;
|
|
308
|
+
}
|
|
309
|
+
|
|
198
310
|
/**
|
|
199
311
|
* Build the fixture `Doc[]` for a topic (0054 Block 4 mapping). Always emits
|
|
200
312
|
* canned docs — the v1 plugin has no live Firecrawl path.
|
|
@@ -244,7 +356,8 @@ export function parseInput(raw: string): InInput {
|
|
|
244
356
|
return { topic: input.topic, maxResults, fixture: input.fixture === true };
|
|
245
357
|
}
|
|
246
358
|
|
|
247
|
-
export async function main(): Promise<number> {
|
|
359
|
+
export async function main(depsOverride?: Partial<SearchDeps>): Promise<number> {
|
|
360
|
+
const fs = createNodeFileSystem();
|
|
248
361
|
const { values } = parseArgs({
|
|
249
362
|
options: {
|
|
250
363
|
in: { type: 'string' },
|
|
@@ -253,34 +366,43 @@ export async function main(): Promise<number> {
|
|
|
253
366
|
});
|
|
254
367
|
|
|
255
368
|
if (!values.in || !values.out) {
|
|
256
|
-
|
|
369
|
+
echoError('Error: Missing required arguments --in and --out');
|
|
257
370
|
return 1;
|
|
258
371
|
}
|
|
259
372
|
|
|
260
373
|
let input: InInput;
|
|
261
374
|
try {
|
|
262
|
-
input = parseInput(await readFile(values.in
|
|
375
|
+
input = parseInput(await fs.readFile(values.in));
|
|
263
376
|
} catch (err: unknown) {
|
|
264
|
-
|
|
377
|
+
echoError(`web-search failed: ${err instanceof Error ? err.message : String(err)}`);
|
|
265
378
|
return 1;
|
|
266
379
|
}
|
|
267
380
|
|
|
268
381
|
if (!input.fixture) {
|
|
269
382
|
// 0054 Block 1/Block 8: `fixture` absent/false = LIVE Firecrawl path.
|
|
270
383
|
try {
|
|
271
|
-
const
|
|
384
|
+
const deps: SearchDeps = {
|
|
272
385
|
fetch: globalThis.fetch,
|
|
273
386
|
env: process.env,
|
|
274
387
|
now: () => new Date(),
|
|
275
|
-
|
|
388
|
+
...depsOverride,
|
|
389
|
+
};
|
|
390
|
+
// Bun.which with an explicit PATH — without the option it can ignore runtime
|
|
391
|
+
// process.env.PATH changes (observed on Bun 1.3.14).
|
|
392
|
+
if (!deps.markitdown && !Bun.which('markitdown', { PATH: deps.env.PATH })) {
|
|
393
|
+
// R6 — live mode requires the normalizer; fail before any network call.
|
|
394
|
+
throw new Error('markitdown is not on PATH');
|
|
395
|
+
}
|
|
396
|
+
const docs = await searchFirecrawl(input, deps);
|
|
397
|
+
const cached = await scrapeAndCacheDocs(docs, deps, values.out);
|
|
276
398
|
const outDir = dirname(values.out);
|
|
277
399
|
if (outDir && outDir !== '.') {
|
|
278
|
-
await
|
|
400
|
+
await fs.ensureDir(outDir);
|
|
279
401
|
}
|
|
280
|
-
await writeFile(values.out, JSON.stringify(DocListSchema.parse(
|
|
402
|
+
await fs.writeFile(values.out, JSON.stringify(DocListSchema.parse(cached), null, 2));
|
|
281
403
|
return 0;
|
|
282
404
|
} catch (err: unknown) {
|
|
283
|
-
|
|
405
|
+
echoError(`web-search failed: ${err instanceof Error ? err.message : String(err)}`);
|
|
284
406
|
return 1;
|
|
285
407
|
}
|
|
286
408
|
}
|
|
@@ -289,16 +411,17 @@ export async function main(): Promise<number> {
|
|
|
289
411
|
const docs = DocListSchema.parse(buildFixtureDocs(input.topic, input.maxResults));
|
|
290
412
|
const outDir = dirname(values.out);
|
|
291
413
|
if (outDir && outDir !== '.') {
|
|
292
|
-
await
|
|
414
|
+
await fs.ensureDir(outDir);
|
|
293
415
|
}
|
|
294
|
-
await writeFile(values.out, JSON.stringify(docs, null, 2)
|
|
416
|
+
await fs.writeFile(values.out, JSON.stringify(docs, null, 2));
|
|
295
417
|
return 0;
|
|
296
418
|
} catch (err: unknown) {
|
|
297
|
-
|
|
419
|
+
echoError(`web-search failed: ${err instanceof Error ? err.message : String(err)}`);
|
|
298
420
|
return 1;
|
|
299
421
|
}
|
|
300
422
|
}
|
|
301
423
|
|
|
302
424
|
if (import.meta.main) {
|
|
425
|
+
await initializeLogger({ console: process.env.NODE_ENV !== 'test', json: true, level: 'warn' });
|
|
303
426
|
process.exit(await main());
|
|
304
427
|
}
|
package/plugins/kk/README.md
CHANGED
|
@@ -7,9 +7,15 @@ It is a Claude Code / Superskill plugin, **not** a kk-core product plugin.
|
|
|
7
7
|
|------|--------|
|
|
8
8
|
| `skills/` | Fat skills (`SKILL.md`) |
|
|
9
9
|
| `commands/` | Thin slash-command wrappers |
|
|
10
|
-
| `agents/` | Thin subagent wrappers |
|
|
10
|
+
| `agents/` | Thin subagent wrappers (currently empty — see below) |
|
|
11
11
|
| `hooks/` | Hook definitions |
|
|
12
12
|
| `rules/` | Agent rules |
|
|
13
|
+
| `workflows/` | Design-time product YAML SSOT (ADR-009). Runtime dest is `$HOME/.config/kk/workflows` via `/workflow-run` |
|
|
14
|
+
|
|
15
|
+
`agents/` is reserved for thin wrappers of fat skills. It is empty: `judge-tech` /
|
|
16
|
+
`judge-tone` / `judge-compliance` were retired (2026-08-19). Rubric selection lives on
|
|
17
|
+
`content-judge` (`agent.run` prompt or `--judge` on `itc-generating`). Do not re-add a
|
|
18
|
+
persona wrapper unless a new skill owns a distinct run procedure.
|
|
13
19
|
|
|
14
20
|
`discoverPlugins` skips this directory. Product plugins (`ingestion` / `generator` /
|
|
15
21
|
`publish`) live under `plugins/ingestions|generations|publishings/<name>/`.
|
|
@@ -17,6 +23,6 @@ It is a Claude Code / Superskill plugin, **not** a kk-core product plugin.
|
|
|
17
23
|
Install: `superskill install kk`.
|
|
18
24
|
|
|
19
25
|
Capability files **omit a leading `kk-`**. `superskill install` prefixes the plugin name, so
|
|
20
|
-
`skills/judge` installs as `kk:judge` (not `kk:kk-judge`). Same for `topic`,
|
|
21
|
-
|
|
26
|
+
`skills/content-judge` installs as `kk:content-judge` (not `kk:kk-content-judge`). Same for `topic`,
|
|
27
|
+
`storm-research`, `itc-generating`, and `/workflow-run`. Product workflow YAML (`kk-storm-research.yaml`)
|
|
22
28
|
keeps its existing name — that is a workflow stem, not an installable agent capability.
|