@gobing-ai/knowledge-kit 0.0.7 → 0.0.9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (61) hide show
  1. package/dist/index.js +21604 -11844
  2. package/package.json +4 -1
  3. package/plugins/generations/content-gen/package.json +2 -1
  4. package/plugins/generations/content-gen/src/index.ts +9 -8
  5. package/plugins/generations/content-gen/src/storm.ts +12 -4
  6. package/plugins/generations/dailynews-gen/package.json +17 -0
  7. package/plugins/generations/dailynews-gen/plugin.json +7 -0
  8. package/plugins/generations/dailynews-gen/src/index.ts +111 -0
  9. package/plugins/generations/dailynews-gen/src/script-builder.ts +121 -0
  10. package/plugins/generations/dailynews-gen/tsconfig.json +4 -0
  11. package/plugins/generations/voice-gen/package.json +17 -0
  12. package/plugins/generations/voice-gen/plugin.json +6 -0
  13. package/plugins/generations/voice-gen/src/concat.ts +218 -0
  14. package/plugins/generations/voice-gen/src/index.ts +228 -0
  15. package/plugins/generations/voice-gen/src/mp3.ts +65 -0
  16. package/plugins/generations/voice-gen/src/voicebox-client.ts +223 -0
  17. package/plugins/generations/voice-gen/src/voicescript.ts +365 -0
  18. package/plugins/generations/voice-gen/tsconfig.json +8 -0
  19. package/plugins/ingestions/aihot-ingest/package.json +17 -0
  20. package/plugins/ingestions/aihot-ingest/plugin.json +7 -0
  21. package/plugins/ingestions/aihot-ingest/src/client.ts +185 -0
  22. package/plugins/ingestions/aihot-ingest/src/index.ts +137 -0
  23. package/plugins/ingestions/aihot-ingest/src/mapper.ts +42 -0
  24. package/plugins/ingestions/aihot-ingest/tsconfig.json +4 -0
  25. package/plugins/ingestions/karakeep-local/package.json +17 -0
  26. package/plugins/ingestions/karakeep-local/src/index.ts +31 -26
  27. package/plugins/ingestions/karakeep-local/tsconfig.json +4 -0
  28. package/plugins/ingestions/web-search/package.json +4 -1
  29. package/plugins/ingestions/web-search/src/index.ts +139 -16
  30. package/plugins/kk/README.md +9 -3
  31. package/plugins/kk/commands/workflow-run.md +100 -28
  32. package/plugins/kk/config.example.yaml +34 -0
  33. package/plugins/kk/scripts/render-md.ts +8 -3
  34. package/plugins/kk/skills/{judge → content-judge}/SKILL.md +13 -14
  35. package/plugins/kk/skills/{judge → content-judge}/references/workflow-integration.md +13 -11
  36. package/plugins/kk/skills/itc-generating/SKILL.md +147 -0
  37. package/plugins/kk/skills/itc-generating/references/generic-craft.md +80 -0
  38. package/plugins/kk/skills/itc-generating/references/platform-english.md +72 -0
  39. package/plugins/kk/skills/itc-generating/references/platform-wechat.md +60 -0
  40. package/plugins/kk/skills/itc-generating/references/skill-authoring.md +62 -0
  41. package/plugins/kk/skills/storm-research/SKILL.md +10 -3
  42. package/plugins/kk/workflows/judge-gated-publish-example.yaml +101 -0
  43. package/plugins/kk/workflows/kk-daily-ai-voice.yaml +144 -0
  44. package/plugins/kk/workflows/kk-ingest-generate-publish.yaml +72 -0
  45. package/plugins/kk/workflows/kk-itc.yaml +285 -0
  46. package/plugins/kk/workflows/kk-solo-podcast.yaml +374 -0
  47. package/plugins/kk/workflows/validate-voicescript.ts +226 -0
  48. package/plugins/publishings/emdash-pub/package.json +17 -0
  49. package/plugins/publishings/emdash-pub/plugin.json +7 -0
  50. package/plugins/publishings/emdash-pub/src/index.ts +450 -0
  51. package/plugins/publishings/emdash-pub/tsconfig.json +4 -0
  52. package/plugins/publishings/qiita-pub/package.json +2 -1
  53. package/plugins/publishings/qiita-pub/src/index.ts +9 -9
  54. package/plugins/publishings/surfdash-pub/package.json +2 -1
  55. package/plugins/publishings/surfdash-pub/src/index.ts +17 -12
  56. package/plugins/publishings/zenn-pub/package.json +2 -1
  57. package/plugins/publishings/zenn-pub/src/index.ts +11 -11
  58. package/plugins/kk/agents/judge-compliance.md +0 -37
  59. package/plugins/kk/agents/judge-tech.md +0 -35
  60. package/plugins/kk/agents/judge-tone.md +0 -37
  61. /package/plugins/kk/skills/{judge → content-judge}/references/rubrics.md +0 -0
@@ -0,0 +1,137 @@
1
+ import { dirname } from 'node:path';
2
+ import { parseArgs } from 'node:util';
3
+ import { type Doc, DocListSchema } from '@gobing-ai/kk-core';
4
+ import { atomicWriteJson, createNodeFileSystem, readJsonFile } from '@gobing-ai/ts-runtime';
5
+ import { echoError } from '@gobing-ai/ts-utils';
6
+ import { z } from 'zod';
7
+ import { type AihotDeps, createAihotClient } from './client';
8
+ import { mapAihotItemsToDocs } from './mapper';
9
+
10
+ export * from './client';
11
+ export * from './mapper';
12
+
13
+ export interface IngestionOptions {
14
+ out: string;
15
+ limit?: number;
16
+ cursor?: string;
17
+ stateFile?: string;
18
+ }
19
+
20
+ export interface IngestionResult {
21
+ docs: Doc[];
22
+ cursor: string;
23
+ asOf?: string;
24
+ }
25
+
26
+ const IngestionInputSchema = z.object({
27
+ limit: z.number().int().min(1).max(100).optional(),
28
+ cursor: z.string().trim().min(1).optional(),
29
+ stateFile: z.string().trim().min(1).optional(),
30
+ });
31
+
32
+ const CursorStateSchema = z.object({ cursor: z.string().trim().min(1) });
33
+
34
+ function parseLimit(value: string): number {
35
+ return z.coerce.number().int().min(1).max(100).parse(value);
36
+ }
37
+
38
+ export async function processIngestionIO(
39
+ options: IngestionOptions,
40
+ depsOverride?: Partial<AihotDeps>,
41
+ ): Promise<IngestionResult> {
42
+ const fs = createNodeFileSystem();
43
+ const client = createAihotClient(depsOverride);
44
+
45
+ let effectiveCursor = options.cursor?.trim();
46
+ if (!effectiveCursor && options.stateFile && (await fs.exists(options.stateFile))) {
47
+ effectiveCursor = CursorStateSchema.parse(await readJsonFile(options.stateFile, fs)).cursor;
48
+ }
49
+
50
+ let items: import('./client').AihotItem[];
51
+ let cursor: string;
52
+ let asOf: string | undefined;
53
+
54
+ if (effectiveCursor) {
55
+ const changes = await client.fetchChanges({
56
+ cursor: effectiveCursor,
57
+ limit: options.limit,
58
+ });
59
+ items = changes.changes.flatMap((change) => (change.op === 'upsert' ? [change.item] : []));
60
+ cursor = changes.cursor;
61
+ } else {
62
+ const snapshot = await client.fetchSnapshot({
63
+ limit: options.limit,
64
+ });
65
+ items = snapshot.items;
66
+ cursor = snapshot.cursor;
67
+ asOf = snapshot.asOf;
68
+ }
69
+
70
+ const docs = DocListSchema.parse(mapAihotItemsToDocs(items));
71
+
72
+ const outDir = dirname(options.out);
73
+ if (outDir && outDir !== '.') {
74
+ await fs.ensureDir(outDir);
75
+ }
76
+ await atomicWriteJson(options.out, docs, fs);
77
+
78
+ if (options.stateFile) {
79
+ const stateDir = dirname(options.stateFile);
80
+ if (stateDir && stateDir !== '.') {
81
+ await fs.ensureDir(stateDir);
82
+ }
83
+ const statePayload = {
84
+ cursor,
85
+ asOf,
86
+ lastSyncAt: new Date().toISOString(),
87
+ itemCount: docs.length,
88
+ };
89
+ await atomicWriteJson(options.stateFile, statePayload, fs);
90
+ }
91
+
92
+ return { docs, cursor, asOf };
93
+ }
94
+
95
+ export async function main(depsOverride?: Partial<AihotDeps>): Promise<number> {
96
+ let values: { in?: string; out?: string; limit?: string; cursor?: string; 'state-file'?: string };
97
+ try {
98
+ ({ values } = parseArgs({
99
+ options: {
100
+ in: { type: 'string' },
101
+ out: { type: 'string' },
102
+ limit: { type: 'string' },
103
+ cursor: { type: 'string' },
104
+ 'state-file': { type: 'string' },
105
+ },
106
+ }));
107
+ } catch (err: unknown) {
108
+ echoError(`aihot-ingest failed: ${err instanceof Error ? err.message : String(err)}`);
109
+ return 1;
110
+ }
111
+
112
+ if (!values.out) {
113
+ echoError('aihot-ingest failed: Missing required argument: --out');
114
+ return 1;
115
+ }
116
+
117
+ try {
118
+ const input = values.in ? IngestionInputSchema.parse(await readJsonFile(values.in)) : {};
119
+ await processIngestionIO(
120
+ {
121
+ out: values.out,
122
+ limit: values.limit ? parseLimit(values.limit) : (input.limit ?? 10),
123
+ cursor: values.cursor ?? input.cursor,
124
+ stateFile: values['state-file'] ?? input.stateFile,
125
+ },
126
+ depsOverride,
127
+ );
128
+ return 0;
129
+ } catch (err: unknown) {
130
+ echoError(`aihot-ingest failed: ${err instanceof Error ? err.message : String(err)}`);
131
+ return 1;
132
+ }
133
+ }
134
+
135
+ if (import.meta.main) {
136
+ process.exit(await main());
137
+ }
@@ -0,0 +1,42 @@
1
+ import { createHash } from 'node:crypto';
2
+ import type { Doc } from '@gobing-ai/kk-core';
3
+ import type { AihotItem } from './client';
4
+
5
+ export function computeDocId(seed: string): string {
6
+ return createHash('sha256').update(seed).digest('hex').slice(0, 16);
7
+ }
8
+
9
+ export function mapAihotItemToDoc(item: AihotItem): Doc {
10
+ const rawId = item.id.trim();
11
+ const canonicalSource = item.links.original || item.links.aihot || item.title || 'untitled';
12
+ const id = rawId.length > 0 ? rawId : computeDocId(canonicalSource);
13
+
14
+ const sourceUri = item.links?.original || item.links?.aihot || `https://aihot.virxact.com/items/${id}`;
15
+ const title = item.title.trim() || 'Untitled AI News';
16
+ const body = `${item.summary?.trim() || item.title.trim()}\n`;
17
+
18
+ const metadata: Record<string, unknown> = {
19
+ sourceName: item.source.name,
20
+ publishedAt: item.publishedAt,
21
+ discoveredAt: item.discoveredAt,
22
+ category: item.category ?? undefined,
23
+ score: item.score ?? undefined,
24
+ aihotUrl: item.links.aihot,
25
+ originalUrl: item.links.original,
26
+ attribution: item.attribution,
27
+ reason: item.reason ?? undefined,
28
+ };
29
+
30
+ return {
31
+ id,
32
+ title,
33
+ sourceUri,
34
+ body,
35
+ mediaType: 'text/markdown',
36
+ metadata,
37
+ };
38
+ }
39
+
40
+ export function mapAihotItemsToDocs(items: AihotItem[]): Doc[] {
41
+ return items.map(mapAihotItemToDoc);
42
+ }
@@ -0,0 +1,4 @@
1
+ {
2
+ "extends": "../../../tooling/typescript/base.json",
3
+ "include": ["src", "tests"]
4
+ }
@@ -0,0 +1,17 @@
1
+ {
2
+ "name": "@gobing-ai/karakeep-local",
3
+ "type": "module",
4
+ "private": true,
5
+ "scripts": {
6
+ "typecheck": "tsc --noEmit"
7
+ },
8
+ "dependencies": {
9
+ "@gobing-ai/kk-core": "workspace:*",
10
+ "@gobing-ai/ts-infra": "catalog:",
11
+ "@gobing-ai/ts-runtime": "catalog:",
12
+ "@gobing-ai/ts-utils": "catalog:"
13
+ },
14
+ "devDependencies": {
15
+ "@types/bun": "1.3.14"
16
+ }
17
+ }
@@ -1,8 +1,12 @@
1
1
  import { createHash } from 'node:crypto';
2
- import { mkdir, readdir, readFile, stat, writeFile } from 'node:fs/promises';
3
- import { dirname, join, relative } from 'node:path';
2
+ import { dirname, relative } from 'node:path';
4
3
  import { parseArgs } from 'node:util';
5
4
  import type { Doc } from '@gobing-ai/kk-core';
5
+ import { getLogger, initializeLogger } from '@gobing-ai/ts-infra';
6
+ import { createNodeFileSystem, walkDir } from '@gobing-ai/ts-runtime';
7
+ import { echoError } from '@gobing-ai/ts-utils';
8
+
9
+ const log = getLogger('kk.ingestion.karakeep-local');
6
10
 
7
11
  interface ParsedMeta {
8
12
  title?: string;
@@ -17,27 +21,26 @@ function computeHash(str: string): string {
17
21
  }
18
22
 
19
23
  async function collectFiles(dir: string): Promise<string[]> {
20
- const entries = await readdir(dir, { recursive: true });
21
- const files: string[] = [];
22
- for (const entry of entries) {
23
- const fullPath = join(dir, entry);
24
- const st = await stat(fullPath);
25
- if (st.isFile()) {
26
- files.push(fullPath);
27
- }
28
- }
29
- return files.sort();
24
+ // walkDir is cycle-safe and root-confined: a directory symlink pointing back at
25
+ // an ancestor is walked at most once, and one escaping `dir` is skipped.
26
+ return (await walkDir(dir, createNodeFileSystem())).sort();
30
27
  }
31
28
 
32
29
  export async function processKarakeepFolder(inDir: string): Promise<Doc[]> {
30
+ const fs = createNodeFileSystem();
33
31
  try {
34
- const inStat = await stat(inDir);
32
+ // The FileSystem seam returns null for a missing path where node's stat throws;
33
+ // both cases must still surface as "Inaccessible source path" below.
34
+ const inStat = await fs.stat(inDir);
35
+ if (inStat === null) {
36
+ throw new Error(`Source path does not exist: ${inDir}`);
37
+ }
35
38
  if (!inStat.isDirectory()) {
36
39
  throw new Error(`Source path is not a directory: ${inDir}`);
37
40
  }
38
41
  } catch (err: unknown) {
39
42
  const msg = `Inaccessible source path: ${inDir}`;
40
- console.error(`Error: ${msg}`, err);
43
+ echoError(`Error: ${msg}`);
41
44
  throw new Error(msg, { cause: err });
42
45
  }
43
46
 
@@ -62,14 +65,14 @@ export async function processKarakeepFolder(inDir: string): Promise<Doc[]> {
62
65
 
63
66
  if (contentMd) {
64
67
  processedFiles.add(contentMd);
65
- const mdText = await readFile(contentMd, 'utf-8');
68
+ const mdText = await fs.readFile(contentMd);
66
69
  let metaObj: ParsedMeta = {};
67
70
  if (metaJson) {
68
71
  processedFiles.add(metaJson);
69
72
  try {
70
- metaObj = JSON.parse(await readFile(metaJson, 'utf-8')) as ParsedMeta;
73
+ metaObj = JSON.parse(await fs.readFile(metaJson)) as ParsedMeta;
71
74
  } catch (err: unknown) {
72
- console.error(`Warning: corrupt/unparsable meta file at ${metaJson}`, err);
75
+ log.warn(`Warning: corrupt/unparsable meta file at ${metaJson}`, { error: err });
73
76
  }
74
77
  }
75
78
 
@@ -99,7 +102,7 @@ export async function processKarakeepFolder(inDir: string): Promise<Doc[]> {
99
102
 
100
103
  if (f.endsWith('.json')) {
101
104
  try {
102
- const jsonText = await readFile(f, 'utf-8');
105
+ const jsonText = await fs.readFile(f);
103
106
  const jsonObj = JSON.parse(jsonText) as ParsedMeta;
104
107
  const { title, sourceUri, mediaType, body, ...extraMeta } = jsonObj;
105
108
  if (typeof body === 'string') {
@@ -112,13 +115,13 @@ export async function processKarakeepFolder(inDir: string): Promise<Doc[]> {
112
115
  metadata: Object.keys(extraMeta).length > 0 ? extraMeta : undefined,
113
116
  });
114
117
  } else {
115
- console.error(`Warning: JSON entry missing string body at ${f}`);
118
+ log.warn(`Warning: JSON entry missing string body at ${f}`);
116
119
  }
117
120
  } catch (err: unknown) {
118
- console.error(`Warning: corrupt/unparsable JSON file at ${f}`, err);
121
+ log.warn(`Warning: corrupt/unparsable JSON file at ${f}`, { error: err });
119
122
  }
120
123
  } else if (f.endsWith('.md')) {
121
- const mdText = await readFile(f, 'utf-8');
124
+ const mdText = await fs.readFile(f);
122
125
  docs.push({
123
126
  id,
124
127
  body: mdText,
@@ -140,31 +143,33 @@ export async function main() {
140
143
  });
141
144
 
142
145
  if (!values.in || !values.out) {
143
- console.error('Error: Missing required arguments --in or --out');
146
+ echoError('Error: Missing required arguments --in or --out');
144
147
  process.exit(1);
145
148
  }
146
149
 
150
+ const fs = createNodeFileSystem();
147
151
  try {
148
152
  const candidateFiles = await collectFiles(values.in);
149
153
  const docs = await processKarakeepFolder(values.in);
150
154
 
151
155
  if (docs.length === 0 && candidateFiles.length > 0) {
152
- console.error(`Fatal: source yielded zero ingestible documents at ${values.in}`);
156
+ echoError(`Fatal: source yielded zero ingestible documents at ${values.in}`);
153
157
  process.exit(1);
154
158
  }
155
159
 
156
160
  const outDir = dirname(values.out);
157
161
  if (outDir && outDir !== '.') {
158
- await mkdir(outDir, { recursive: true });
162
+ await fs.ensureDir(outDir);
159
163
  }
160
164
 
161
- await writeFile(values.out, JSON.stringify(docs, null, 2), 'utf-8');
165
+ await fs.writeFile(values.out, JSON.stringify(docs, null, 2));
162
166
  } catch (err: unknown) {
163
- console.error('Fatal error during karakeep-local ingestion:', err);
167
+ echoError(`Fatal error during karakeep-local ingestion: ${err instanceof Error ? err.message : String(err)}`);
164
168
  process.exit(1);
165
169
  }
166
170
  }
167
171
 
168
172
  if (import.meta.main) {
173
+ await initializeLogger({ console: process.env.NODE_ENV !== 'test', json: true, level: 'warn' });
169
174
  main();
170
175
  }
@@ -0,0 +1,4 @@
1
+ {
2
+ "extends": "../../../tooling/typescript/base.json",
3
+ "include": ["src", "tests"]
4
+ }
@@ -6,7 +6,10 @@
6
6
  "typecheck": "tsc --noEmit"
7
7
  },
8
8
  "dependencies": {
9
- "@gobing-ai/kk-core": "workspace:*"
9
+ "@gobing-ai/kk-core": "workspace:*",
10
+ "@gobing-ai/ts-infra": "catalog:",
11
+ "@gobing-ai/ts-runtime": "catalog:",
12
+ "@gobing-ai/ts-utils": "catalog:"
10
13
  },
11
14
  "devDependencies": {
12
15
  "@types/bun": "1.3.14"
@@ -1,8 +1,12 @@
1
1
  import { createHash } from 'node:crypto';
2
- import { mkdir, readFile, writeFile } from 'node:fs/promises';
3
- import { dirname } from 'node:path';
2
+ import { dirname, join } from 'node:path';
4
3
  import { parseArgs } from 'node:util';
5
4
  import { type Doc, DocListSchema } from '@gobing-ai/kk-core';
5
+ import { getLogger, initializeLogger } from '@gobing-ai/ts-infra';
6
+ import { createNodeFileSystem } from '@gobing-ai/ts-runtime';
7
+ import { echoError } from '@gobing-ai/ts-utils';
8
+
9
+ const log = getLogger('kk.ingestion.web-search');
6
10
 
7
11
  /** Validated `--in` input per the 0054 contract (`topic` required; `maxResults`/`fixture` optional). */
8
12
  interface InInput {
@@ -49,13 +53,49 @@ export function computeId(uri: string): string {
49
53
  /** Fetch signature the live path accepts, so tests can inject a fake (CI never hits the network). */
50
54
  export type FetchLike = (input: string, init?: RequestInit) => Promise<Response>;
51
55
 
52
- /** Live-path dependencies — fetch, env, and clock are all injectable for tests. */
56
+ /** markitdown normalizer — HTML in, markdown out (ADR-013 D3). Throws on non-zero exit or
57
+ * empty output; callers treat a throw as unusable content (per-URL fallback, D5). */
58
+ export type MarkitdownRunner = (html: string) => Promise<string>;
59
+
60
+ /** Live-path dependencies — fetch, env, clock, scrape, and markitdown are all injectable for tests. */
53
61
  export interface SearchDeps {
54
62
  fetch: FetchLike;
55
63
  env: Record<string, string | undefined>;
56
64
  now: () => Date;
65
+ /** Scrape transport; defaults to `deps.fetch` so tests can route both Firecrawl calls separately. */
66
+ scrape?: FetchLike;
67
+ /** markitdown runner; when absent in live mode, `main` probes PATH and fails loud (R6). */
68
+ markitdown?: MarkitdownRunner;
57
69
  }
58
70
 
71
+ /** Real markitdown runner: `markitdown -x .html`, HTML on stdin, markdown on stdout. The
72
+ * `-x .html` extension hint is required — bare stdin passes HTML through unconverted.
73
+ * Spawns the absolute path resolved via Bun.which (with an explicit PATH option — plain
74
+ * argv[0] lookup can resolve against a startup PATH snapshot, missing runtime edits). */
75
+ export const realMarkitdown: MarkitdownRunner = async (html: string): Promise<string> => {
76
+ const bin = Bun.which('markitdown', { PATH: process.env.PATH });
77
+ if (!bin) {
78
+ throw new Error('markitdown is not on PATH');
79
+ }
80
+ const proc = Bun.spawn([bin, '-x', '.html'], { stdin: 'pipe', stdout: 'pipe', stderr: 'pipe' });
81
+ proc.stdin.write(html);
82
+ await proc.stdin.end();
83
+ const [stdout, stderrText, exitCode] = await Promise.all([
84
+ new Response(proc.stdout).text(),
85
+ new Response(proc.stderr).text(),
86
+ proc.exited,
87
+ ]);
88
+ if (exitCode !== 0) {
89
+ const detail = stderrText.trim();
90
+ throw new Error(`markitdown exited ${exitCode}${detail ? `: ${detail}` : ''}`);
91
+ }
92
+ const markdown = stdout.trim();
93
+ if (markdown.length === 0) {
94
+ throw new Error('markitdown produced empty output');
95
+ }
96
+ return markdown;
97
+ };
98
+
59
99
  /** One Firecrawl `data.web[]` item — the subset of fields 0054 Block 3/4 consume. */
60
100
  interface FirecrawlWebItem {
61
101
  url?: unknown;
@@ -70,6 +110,12 @@ interface FirecrawlSearchResponse {
70
110
  data?: { web?: FirecrawlWebItem[] | null } | null;
71
111
  }
72
112
 
113
+ /** One Firecrawl `/v2/scrape` body — the subset the cache pass consumes. */
114
+ interface FirecrawlScrapeResponse {
115
+ success?: unknown;
116
+ data?: { html?: unknown; markdown?: unknown } | null;
117
+ }
118
+
73
119
  /**
74
120
  * Live Firecrawl search → Doc[] (0054 Blocks 2–5). Throws an Error whose message is the
75
121
  * fail-loud diagnostic; `main` prefixes it with `web-search failed: ` and returns 1.
@@ -152,7 +198,7 @@ export async function searchFirecrawl(input: InInput, deps: SearchDeps): Promise
152
198
  if (!url || body.length === 0) {
153
199
  // Row 7 — partial item failure: skip + stderr warn, continue.
154
200
  skipped += 1;
155
- console.warn(`web-search: skipping result without usable markdown (url=${url ?? 'missing'})`);
201
+ log.warn(`web-search: skipping result without usable markdown (url=${url ?? 'missing'})`);
156
202
  continue;
157
203
  }
158
204
  docs.push({
@@ -195,6 +241,72 @@ function extractError(bodyText: string): string {
195
241
  return '';
196
242
  }
197
243
 
244
+ /** Scrape + normalize + cache pass over the search results (ADR-013 D1–D6). Sequential
245
+ * per Doc — deliberate, no parallel fan-out (rate-limit friendly, deterministic). Cache hit
246
+ * (`references/<doc-id>.md` exists) reuses the file with no new /v2/scrape call (R3/R4);
247
+ * otherwise scrape → normalize (html via markitdown, markdown as-is) → write cache → body
248
+ * swap. Per-URL failure or unusable content warns and keeps the snippet body (R5/R11) — never
249
+ * fatal. Returns a new Doc[]; input order and non-body fields are untouched. */
250
+ export async function scrapeAndCacheDocs(docs: Doc[], deps: SearchDeps, outPath: string): Promise<Doc[]> {
251
+ const fs = createNodeFileSystem();
252
+ const cacheDir = join(dirname(outPath), 'references');
253
+ const scrapeFetch = deps.scrape ?? deps.fetch;
254
+ const out: Doc[] = [];
255
+ for (const doc of docs) {
256
+ const uri = doc.sourceUri;
257
+ if (!uri) {
258
+ out.push(doc);
259
+ continue;
260
+ }
261
+ const cacheFile = join(cacheDir, `${doc.id}.md`);
262
+ try {
263
+ const cached = await fs.readFile(cacheFile);
264
+ if (cached.trim().length > 0) {
265
+ out.push({ ...doc, body: cached });
266
+ continue;
267
+ }
268
+ } catch {
269
+ // Cache miss — fall through to the live scrape.
270
+ }
271
+ try {
272
+ const key = deps.env.FIRECRAWL_API_KEY;
273
+ if (!key) throw new Error('FIRECRAWL_API_KEY is unset');
274
+ const res = await scrapeFetch('https://api.firecrawl.dev/v2/scrape', {
275
+ method: 'POST',
276
+ headers: {
277
+ 'Content-Type': 'application/json',
278
+ Authorization: `Bearer ${key}`,
279
+ },
280
+ body: JSON.stringify({ url: uri, formats: ['html', 'markdown'] }),
281
+ });
282
+ if (!res.ok || res.status !== 200) {
283
+ throw new Error(`Firecrawl HTTP ${res.status}`);
284
+ }
285
+ const parsed = JSON.parse(await res.text()) as FirecrawlScrapeResponse;
286
+ if (parsed.success !== true || parsed.data === null || parsed.data === undefined) {
287
+ throw new Error('Firecrawl malformed scrape response');
288
+ }
289
+ const html = typeof parsed.data.html === 'string' ? parsed.data.html : undefined;
290
+ const markdown = typeof parsed.data.markdown === 'string' ? parsed.data.markdown : undefined;
291
+ let normalized: string;
292
+ if (html && html.trim().length > 0) {
293
+ normalized = await (deps.markitdown ?? realMarkitdown)(html);
294
+ } else if (markdown && markdown.trim().length > 0) {
295
+ normalized = markdown.trim();
296
+ } else {
297
+ throw new Error('scrape returned no usable content');
298
+ }
299
+ await fs.ensureDir(cacheDir);
300
+ await fs.writeFile(cacheFile, normalized);
301
+ out.push({ ...doc, body: normalized });
302
+ } catch (err: unknown) {
303
+ log.warn(`web-search: scrape failed (url=${uri}): ${err instanceof Error ? err.message : String(err)}`);
304
+ out.push(doc);
305
+ }
306
+ }
307
+ return out;
308
+ }
309
+
198
310
  /**
199
311
  * Build the fixture `Doc[]` for a topic (0054 Block 4 mapping). Always emits
200
312
  * canned docs — the v1 plugin has no live Firecrawl path.
@@ -244,7 +356,8 @@ export function parseInput(raw: string): InInput {
244
356
  return { topic: input.topic, maxResults, fixture: input.fixture === true };
245
357
  }
246
358
 
247
- export async function main(): Promise<number> {
359
+ export async function main(depsOverride?: Partial<SearchDeps>): Promise<number> {
360
+ const fs = createNodeFileSystem();
248
361
  const { values } = parseArgs({
249
362
  options: {
250
363
  in: { type: 'string' },
@@ -253,34 +366,43 @@ export async function main(): Promise<number> {
253
366
  });
254
367
 
255
368
  if (!values.in || !values.out) {
256
- console.error('Error: Missing required arguments --in and --out');
369
+ echoError('Error: Missing required arguments --in and --out');
257
370
  return 1;
258
371
  }
259
372
 
260
373
  let input: InInput;
261
374
  try {
262
- input = parseInput(await readFile(values.in, 'utf-8'));
375
+ input = parseInput(await fs.readFile(values.in));
263
376
  } catch (err: unknown) {
264
- console.error(`web-search failed: ${err instanceof Error ? err.message : String(err)}`);
377
+ echoError(`web-search failed: ${err instanceof Error ? err.message : String(err)}`);
265
378
  return 1;
266
379
  }
267
380
 
268
381
  if (!input.fixture) {
269
382
  // 0054 Block 1/Block 8: `fixture` absent/false = LIVE Firecrawl path.
270
383
  try {
271
- const docs = await searchFirecrawl(input, {
384
+ const deps: SearchDeps = {
272
385
  fetch: globalThis.fetch,
273
386
  env: process.env,
274
387
  now: () => new Date(),
275
- });
388
+ ...depsOverride,
389
+ };
390
+ // Bun.which with an explicit PATH — without the option it can ignore runtime
391
+ // process.env.PATH changes (observed on Bun 1.3.14).
392
+ if (!deps.markitdown && !Bun.which('markitdown', { PATH: deps.env.PATH })) {
393
+ // R6 — live mode requires the normalizer; fail before any network call.
394
+ throw new Error('markitdown is not on PATH');
395
+ }
396
+ const docs = await searchFirecrawl(input, deps);
397
+ const cached = await scrapeAndCacheDocs(docs, deps, values.out);
276
398
  const outDir = dirname(values.out);
277
399
  if (outDir && outDir !== '.') {
278
- await mkdir(outDir, { recursive: true });
400
+ await fs.ensureDir(outDir);
279
401
  }
280
- await writeFile(values.out, JSON.stringify(DocListSchema.parse(docs), null, 2), 'utf-8');
402
+ await fs.writeFile(values.out, JSON.stringify(DocListSchema.parse(cached), null, 2));
281
403
  return 0;
282
404
  } catch (err: unknown) {
283
- console.error(`web-search failed: ${err instanceof Error ? err.message : String(err)}`);
405
+ echoError(`web-search failed: ${err instanceof Error ? err.message : String(err)}`);
284
406
  return 1;
285
407
  }
286
408
  }
@@ -289,16 +411,17 @@ export async function main(): Promise<number> {
289
411
  const docs = DocListSchema.parse(buildFixtureDocs(input.topic, input.maxResults));
290
412
  const outDir = dirname(values.out);
291
413
  if (outDir && outDir !== '.') {
292
- await mkdir(outDir, { recursive: true });
414
+ await fs.ensureDir(outDir);
293
415
  }
294
- await writeFile(values.out, JSON.stringify(docs, null, 2), 'utf-8');
416
+ await fs.writeFile(values.out, JSON.stringify(docs, null, 2));
295
417
  return 0;
296
418
  } catch (err: unknown) {
297
- console.error(`web-search failed: ${err instanceof Error ? err.message : String(err)}`);
419
+ echoError(`web-search failed: ${err instanceof Error ? err.message : String(err)}`);
298
420
  return 1;
299
421
  }
300
422
  }
301
423
 
302
424
  if (import.meta.main) {
425
+ await initializeLogger({ console: process.env.NODE_ENV !== 'test', json: true, level: 'warn' });
303
426
  process.exit(await main());
304
427
  }
@@ -7,9 +7,15 @@ It is a Claude Code / Superskill plugin, **not** a kk-core product plugin.
7
7
  |------|--------|
8
8
  | `skills/` | Fat skills (`SKILL.md`) |
9
9
  | `commands/` | Thin slash-command wrappers |
10
- | `agents/` | Thin subagent wrappers |
10
+ | `agents/` | Thin subagent wrappers (currently empty — see below) |
11
11
  | `hooks/` | Hook definitions |
12
12
  | `rules/` | Agent rules |
13
+ | `workflows/` | Design-time product YAML SSOT (ADR-009). Runtime dest is `$HOME/.config/kk/workflows` via `/workflow-run` |
14
+
15
+ `agents/` is reserved for thin wrappers of fat skills. It is empty: `judge-tech` /
16
+ `judge-tone` / `judge-compliance` were retired (2026-08-19). Rubric selection lives on
17
+ `content-judge` (`agent.run` prompt or `--judge` on `itc-generating`). Do not re-add a
18
+ persona wrapper unless a new skill owns a distinct run procedure.
13
19
 
14
20
  `discoverPlugins` skips this directory. Product plugins (`ingestion` / `generator` /
15
21
  `publish`) live under `plugins/ingestions|generations|publishings/<name>/`.
@@ -17,6 +23,6 @@ It is a Claude Code / Superskill plugin, **not** a kk-core product plugin.
17
23
  Install: `superskill install kk`.
18
24
 
19
25
  Capability files **omit a leading `kk-`**. `superskill install` prefixes the plugin name, so
20
- `skills/judge` installs as `kk:judge` (not `kk:kk-judge`). Same for `topic`, `storm-research`,
21
- `/workflow-run`, and the `judge-*` subagents. Product workflow YAML (`kk-storm-research.yaml`)
26
+ `skills/content-judge` installs as `kk:content-judge` (not `kk:kk-content-judge`). Same for `topic`,
27
+ `storm-research`, `itc-generating`, and `/workflow-run`. Product workflow YAML (`kk-storm-research.yaml`)
22
28
  keeps its existing name — that is a workflow stem, not an installable agent capability.