@crowdlisten/harness 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (109) hide show
  1. package/AGENTS.md +167 -0
  2. package/LICENSE +21 -0
  3. package/README.md +153 -0
  4. package/dist/agent-proxy.d.ts +24 -0
  5. package/dist/agent-proxy.js +140 -0
  6. package/dist/agent-tools.d.ts +736 -0
  7. package/dist/agent-tools.js +409 -0
  8. package/dist/context/api.d.ts +5 -0
  9. package/dist/context/api.js +164 -0
  10. package/dist/context/cli.d.ts +19 -0
  11. package/dist/context/cli.js +108 -0
  12. package/dist/context/extractor.d.ts +12 -0
  13. package/dist/context/extractor.js +43 -0
  14. package/dist/context/index.d.ts +12 -0
  15. package/dist/context/index.js +11 -0
  16. package/dist/context/matcher.d.ts +39 -0
  17. package/dist/context/matcher.js +246 -0
  18. package/dist/context/parser.d.ts +28 -0
  19. package/dist/context/parser.js +157 -0
  20. package/dist/context/pipeline.d.ts +26 -0
  21. package/dist/context/pipeline.js +56 -0
  22. package/dist/context/prompts.d.ts +6 -0
  23. package/dist/context/prompts.js +60 -0
  24. package/dist/context/providers.d.ts +6 -0
  25. package/dist/context/providers.js +106 -0
  26. package/dist/context/redactor.d.ts +10 -0
  27. package/dist/context/redactor.js +68 -0
  28. package/dist/context/server.d.ts +5 -0
  29. package/dist/context/server.js +134 -0
  30. package/dist/context/store.d.ts +12 -0
  31. package/dist/context/store.js +82 -0
  32. package/dist/context/types.d.ts +79 -0
  33. package/dist/context/types.js +4 -0
  34. package/dist/context/user-state.d.ts +40 -0
  35. package/dist/context/user-state.js +144 -0
  36. package/dist/index.d.ts +14 -0
  37. package/dist/index.js +385 -0
  38. package/dist/insights/browser/BrowserPool.d.ts +87 -0
  39. package/dist/insights/browser/BrowserPool.js +266 -0
  40. package/dist/insights/browser/RequestInterceptor.d.ts +46 -0
  41. package/dist/insights/browser/RequestInterceptor.js +115 -0
  42. package/dist/insights/cli.d.ts +8 -0
  43. package/dist/insights/cli.js +206 -0
  44. package/dist/insights/core/base/BaseAdapter.d.ts +37 -0
  45. package/dist/insights/core/base/BaseAdapter.js +123 -0
  46. package/dist/insights/core/health/HealthMonitor.d.ts +75 -0
  47. package/dist/insights/core/health/HealthMonitor.js +171 -0
  48. package/dist/insights/core/interfaces/SocialMediaPlatform.d.ts +125 -0
  49. package/dist/insights/core/interfaces/SocialMediaPlatform.js +42 -0
  50. package/dist/insights/core/utils/DataNormalizer.d.ts +53 -0
  51. package/dist/insights/core/utils/DataNormalizer.js +349 -0
  52. package/dist/insights/core/utils/InstagramUrlUtils.d.ts +11 -0
  53. package/dist/insights/core/utils/InstagramUrlUtils.js +60 -0
  54. package/dist/insights/core/utils/TikTokUrlUtils.d.ts +10 -0
  55. package/dist/insights/core/utils/TikTokUrlUtils.js +57 -0
  56. package/dist/insights/handlers.d.ts +157 -0
  57. package/dist/insights/handlers.js +246 -0
  58. package/dist/insights/index.d.ts +437 -0
  59. package/dist/insights/index.js +426 -0
  60. package/dist/insights/platforms/instagram/InstagramAdapter.d.ts +34 -0
  61. package/dist/insights/platforms/instagram/InstagramAdapter.js +342 -0
  62. package/dist/insights/platforms/moltbook/MoltbookAdapter.d.ts +31 -0
  63. package/dist/insights/platforms/moltbook/MoltbookAdapter.js +227 -0
  64. package/dist/insights/platforms/reddit/RedditAdapter.d.ts +21 -0
  65. package/dist/insights/platforms/reddit/RedditAdapter.js +212 -0
  66. package/dist/insights/platforms/tiktok/TikTokAdapter.d.ts +34 -0
  67. package/dist/insights/platforms/tiktok/TikTokAdapter.js +269 -0
  68. package/dist/insights/platforms/twitter/TwitterAdapter.d.ts +23 -0
  69. package/dist/insights/platforms/twitter/TwitterAdapter.js +211 -0
  70. package/dist/insights/platforms/xiaohongshu/XiaohongshuAdapter.d.ts +35 -0
  71. package/dist/insights/platforms/xiaohongshu/XiaohongshuAdapter.js +258 -0
  72. package/dist/insights/platforms/youtube/YouTubeAdapter.d.ts +22 -0
  73. package/dist/insights/platforms/youtube/YouTubeAdapter.js +254 -0
  74. package/dist/insights/service-config.d.ts +7 -0
  75. package/dist/insights/service-config.js +60 -0
  76. package/dist/insights/services/UnifiedSocialMediaService.d.ts +94 -0
  77. package/dist/insights/services/UnifiedSocialMediaService.js +259 -0
  78. package/dist/insights/vision/VisionExtractor.d.ts +46 -0
  79. package/dist/insights/vision/VisionExtractor.js +236 -0
  80. package/dist/learnings.d.ts +50 -0
  81. package/dist/learnings.js +130 -0
  82. package/dist/openapi.d.ts +29 -0
  83. package/dist/openapi.js +169 -0
  84. package/dist/server-factory.d.ts +20 -0
  85. package/dist/server-factory.js +41 -0
  86. package/dist/suggestions.d.ts +16 -0
  87. package/dist/suggestions.js +72 -0
  88. package/dist/telemetry.d.ts +44 -0
  89. package/dist/telemetry.js +93 -0
  90. package/dist/tools/registry.d.ts +65 -0
  91. package/dist/tools/registry.js +256 -0
  92. package/dist/tools.d.ts +2433 -0
  93. package/dist/tools.js +2294 -0
  94. package/dist/transport/http.d.ts +15 -0
  95. package/dist/transport/http.js +154 -0
  96. package/package.json +76 -0
  97. package/skills/catalog.json +272 -0
  98. package/skills/community-catalog.json +4202 -0
  99. package/skills/competitive-analysis/SKILL.md +174 -0
  100. package/skills/content-creator/SKILL.md +256 -0
  101. package/skills/content-strategy/SKILL.md +222 -0
  102. package/skills/data-storytelling/SKILL.md +248 -0
  103. package/skills/heuristic-evaluation/SKILL.md +201 -0
  104. package/skills/market-research-reports/SKILL.md +184 -0
  105. package/skills/user-stories/SKILL.md +178 -0
  106. package/skills/ux-researcher/SKILL.md +239 -0
  107. package/web-dist/assets/index-B1b25lNd.css +1 -0
  108. package/web-dist/assets/index-CDWHwHbl.js +64 -0
  109. package/web-dist/index.html +16 -0
@@ -0,0 +1,56 @@
1
+ /**
2
+ * Orchestrator: parse -> redact -> chunk -> extract -> match -> store
3
+ */
4
+ import { redactPII } from "./redactor.js";
5
+ import { parseFile, chunkText } from "./parser.js";
6
+ import { extractBlocks } from "./extractor.js";
7
+ import { matchSkills } from "./matcher.js";
8
+ import { createProvider } from "./providers.js";
9
+ import { addBlocks, loadConfig } from "./store.js";
10
+ /**
11
+ * Run the full context extraction pipeline.
12
+ */
13
+ export async function runPipeline(opts) {
14
+ const config = opts.config || loadConfig();
15
+ if (!config) {
16
+ throw new Error("No LLM provider configured. Run: npx @crowdlisten/harness setup");
17
+ }
18
+ const provider = createProvider(config);
19
+ const source = opts.source || opts.filePath || "paste";
20
+ // 1. Get raw text
21
+ let rawText = opts.text || "";
22
+ if (opts.filePath) {
23
+ const parsed = await parseFile(opts.filePath);
24
+ rawText = rawText ? rawText + "\n\n---\n\n" + parsed : parsed;
25
+ }
26
+ if (!rawText.trim()) {
27
+ throw new Error("No content to process");
28
+ }
29
+ // 2. PII redaction
30
+ const { redactedText, stats: redactionStats, totalRedactions } = redactPII(rawText);
31
+ // 3. Chunk
32
+ const chunks = chunkText(redactedText);
33
+ const total = chunks.length;
34
+ // 4. Extract blocks from each chunk
35
+ const allBlocks = [];
36
+ const isChat = opts.isChat !== false; // default true
37
+ for (let i = 0; i < chunks.length; i++) {
38
+ opts.onProgress?.({ current: i + 1, total, blocksFound: allBlocks.length });
39
+ const blocks = await extractBlocks(provider, chunks[i], source, isChat);
40
+ allBlocks.push(...blocks);
41
+ }
42
+ // 5. Match skills
43
+ const skills = await matchSkills(allBlocks);
44
+ // 6. Store locally
45
+ if (allBlocks.length > 0) {
46
+ addBlocks(allBlocks, source);
47
+ }
48
+ opts.onProgress?.({ current: total, total, blocksFound: allBlocks.length });
49
+ return {
50
+ blocks: allBlocks,
51
+ skills,
52
+ redactionStats,
53
+ totalRedactions,
54
+ chunkCount: total,
55
+ };
56
+ }
@@ -0,0 +1,6 @@
1
+ /**
2
+ * System prompts for LLM-based context extraction.
3
+ * Extracted from crowdlisten_deployed/frontend/src/services/contextBlockService.js
4
+ */
5
+ export declare const EXTRACT_SYSTEM_PROMPT = "You are a context extraction engine. Analyze the provided text and extract reusable context blocks.\n\nExtract two categories:\n1. **style** \u2014 Writing patterns, tone, communication style, formatting habits, voice characteristics\n2. **insight** \u2014 Key facts, quotes, data points, actionable knowledge, domain expertise\n\nReturn JSON with this exact schema:\n{\n \"blocks\": [\n {\n \"type\": \"style\" | \"insight\",\n \"title\": \"Short descriptive title (3-8 words)\",\n \"content\": \"The extracted context block content (1-3 sentences)\"\n }\n ]\n}\n\nRules:\n- Extract 3-10 blocks depending on content richness\n- Each block should be self-contained and reusable\n- Style blocks capture HOW the person communicates\n- Insight blocks capture WHAT knowledge is present\n- Titles should be specific and descriptive\n- Content should be concise but complete\n- Return valid JSON only, no markdown fences";
6
+ export declare const CHAT_EXTRACT_SYSTEM_PROMPT = "You are a context extraction engine analyzing AI chat history. Extract reusable context blocks that capture the user's patterns, knowledge, and preferences.\n\nExtract four categories:\n1. **style** \u2014 Writing patterns, tone, communication style, formatting habits, voice characteristics\n2. **insight** \u2014 Key facts, domain expertise, actionable knowledge, important decisions or conclusions\n3. **pattern** \u2014 Recurring workflows, decision-making approaches, problem-solving methods, repeated processes\n4. **preference** \u2014 Tool choices, format preferences, working style, technology opinions, environment setup\n\nReturn JSON with this exact schema:\n{\n \"blocks\": [\n {\n \"type\": \"style\" | \"insight\" | \"pattern\" | \"preference\",\n \"title\": \"Short descriptive title (3-8 words)\",\n \"content\": \"The extracted context block content (1-3 sentences)\"\n }\n ]\n}\n\nRules:\n- Extract 5-15 blocks depending on content richness\n- Each block should be self-contained and reusable across conversations\n- Deduplicate \u2014 if the same theme appears multiple times, synthesize into one block\n- Focus on the USER's behavior, not the assistant's\n- Style blocks capture HOW the person communicates\n- Insight blocks capture WHAT knowledge or expertise they have\n- Pattern blocks capture HOW they work and make decisions\n- Preference blocks capture WHAT tools/formats/approaches they choose\n- Titles should be specific and descriptive\n- Content should be concise but complete\n- Return valid JSON only, no markdown fences";
@@ -0,0 +1,60 @@
1
+ /**
2
+ * System prompts for LLM-based context extraction.
3
+ * Extracted from crowdlisten_deployed/frontend/src/services/contextBlockService.js
4
+ */
5
+ export const EXTRACT_SYSTEM_PROMPT = `You are a context extraction engine. Analyze the provided text and extract reusable context blocks.
6
+
7
+ Extract two categories:
8
+ 1. **style** — Writing patterns, tone, communication style, formatting habits, voice characteristics
9
+ 2. **insight** — Key facts, quotes, data points, actionable knowledge, domain expertise
10
+
11
+ Return JSON with this exact schema:
12
+ {
13
+ "blocks": [
14
+ {
15
+ "type": "style" | "insight",
16
+ "title": "Short descriptive title (3-8 words)",
17
+ "content": "The extracted context block content (1-3 sentences)"
18
+ }
19
+ ]
20
+ }
21
+
22
+ Rules:
23
+ - Extract 3-10 blocks depending on content richness
24
+ - Each block should be self-contained and reusable
25
+ - Style blocks capture HOW the person communicates
26
+ - Insight blocks capture WHAT knowledge is present
27
+ - Titles should be specific and descriptive
28
+ - Content should be concise but complete
29
+ - Return valid JSON only, no markdown fences`;
30
+ export const CHAT_EXTRACT_SYSTEM_PROMPT = `You are a context extraction engine analyzing AI chat history. Extract reusable context blocks that capture the user's patterns, knowledge, and preferences.
31
+
32
+ Extract four categories:
33
+ 1. **style** — Writing patterns, tone, communication style, formatting habits, voice characteristics
34
+ 2. **insight** — Key facts, domain expertise, actionable knowledge, important decisions or conclusions
35
+ 3. **pattern** — Recurring workflows, decision-making approaches, problem-solving methods, repeated processes
36
+ 4. **preference** — Tool choices, format preferences, working style, technology opinions, environment setup
37
+
38
+ Return JSON with this exact schema:
39
+ {
40
+ "blocks": [
41
+ {
42
+ "type": "style" | "insight" | "pattern" | "preference",
43
+ "title": "Short descriptive title (3-8 words)",
44
+ "content": "The extracted context block content (1-3 sentences)"
45
+ }
46
+ ]
47
+ }
48
+
49
+ Rules:
50
+ - Extract 5-15 blocks depending on content richness
51
+ - Each block should be self-contained and reusable across conversations
52
+ - Deduplicate — if the same theme appears multiple times, synthesize into one block
53
+ - Focus on the USER's behavior, not the assistant's
54
+ - Style blocks capture HOW the person communicates
55
+ - Insight blocks capture WHAT knowledge or expertise they have
56
+ - Pattern blocks capture HOW they work and make decisions
57
+ - Preference blocks capture WHAT tools/formats/approaches they choose
58
+ - Titles should be specific and descriptive
59
+ - Content should be concise but complete
60
+ - Return valid JSON only, no markdown fences`;
@@ -0,0 +1,6 @@
1
+ /**
2
+ * LLM provider abstraction: OpenAI, Anthropic, Ollama.
3
+ * Uses the user's own API key — configured via `npx @crowdlisten/harness setup`.
4
+ */
5
+ import type { LLMProvider, ContextConfig } from "./types.js";
6
+ export declare function createProvider(config: ContextConfig): LLMProvider;
@@ -0,0 +1,106 @@
1
+ /**
2
+ * LLM provider abstraction: OpenAI, Anthropic, Ollama.
3
+ * Uses the user's own API key — configured via `npx @crowdlisten/harness setup`.
4
+ */
5
+ // ─── OpenAI ──────────────────────────────────────────────────────────────────
6
+ class OpenAIProvider {
7
+ name = "openai";
8
+ apiKey;
9
+ defaultModel;
10
+ constructor(apiKey, model) {
11
+ this.apiKey = apiKey;
12
+ this.defaultModel = model || "gpt-4o-mini";
13
+ }
14
+ async complete(messages, opts) {
15
+ const body = {
16
+ model: opts?.model || this.defaultModel,
17
+ messages,
18
+ temperature: opts?.temperature ?? 0.3,
19
+ };
20
+ if (opts?.maxTokens)
21
+ body.max_tokens = opts.maxTokens;
22
+ if (opts?.jsonMode)
23
+ body.response_format = { type: "json_object" };
24
+ const res = await fetch("https://api.openai.com/v1/chat/completions", {
25
+ method: "POST",
26
+ headers: {
27
+ "Content-Type": "application/json",
28
+ Authorization: `Bearer ${this.apiKey}`,
29
+ },
30
+ body: JSON.stringify(body),
31
+ });
32
+ if (!res.ok) {
33
+ const err = await res.json().catch(() => ({}));
34
+ throw new Error(err.error?.message || `OpenAI request failed (${res.status})`);
35
+ }
36
+ const data = (await res.json());
37
+ return data.choices?.[0]?.message?.content || "";
38
+ }
39
+ async embed(texts) {
40
+ const res = await fetch("https://api.openai.com/v1/embeddings", {
41
+ method: "POST",
42
+ headers: {
43
+ "Content-Type": "application/json",
44
+ Authorization: `Bearer ${this.apiKey}`,
45
+ },
46
+ body: JSON.stringify({
47
+ model: "text-embedding-3-small",
48
+ input: texts,
49
+ }),
50
+ });
51
+ if (!res.ok) {
52
+ throw new Error(`OpenAI embeddings failed (${res.status})`);
53
+ }
54
+ const data = (await res.json());
55
+ return data.data.map((d) => d.embedding);
56
+ }
57
+ }
58
+ // ─── Anthropic ───────────────────────────────────────────────────────────────
59
+ class AnthropicProvider {
60
+ name = "anthropic";
61
+ apiKey;
62
+ defaultModel;
63
+ constructor(apiKey, model) {
64
+ this.apiKey = apiKey;
65
+ this.defaultModel = model || "claude-sonnet-4-20250514";
66
+ }
67
+ async complete(messages, opts) {
68
+ const systemMsg = messages.find((m) => m.role === "system");
69
+ const nonSystem = messages.filter((m) => m.role !== "system");
70
+ const body = {
71
+ model: opts?.model || this.defaultModel,
72
+ max_tokens: opts?.maxTokens || 4096,
73
+ messages: nonSystem.map((m) => ({ role: m.role, content: m.content })),
74
+ };
75
+ if (systemMsg)
76
+ body.system = systemMsg.content;
77
+ const res = await fetch("https://api.anthropic.com/v1/messages", {
78
+ method: "POST",
79
+ headers: {
80
+ "Content-Type": "application/json",
81
+ "x-api-key": this.apiKey,
82
+ "anthropic-version": "2023-06-01",
83
+ },
84
+ body: JSON.stringify(body),
85
+ });
86
+ if (!res.ok) {
87
+ const err = await res.json().catch(() => ({}));
88
+ throw new Error(err.error?.message ||
89
+ `Anthropic request failed (${res.status})`);
90
+ }
91
+ const data = (await res.json());
92
+ const textBlock = data.content?.find((c) => c.type === "text");
93
+ return textBlock?.text || "";
94
+ }
95
+ }
96
+ // ─── Factory ─────────────────────────────────────────────────────────────────
97
+ export function createProvider(config) {
98
+ switch (config.provider) {
99
+ case "openai":
100
+ return new OpenAIProvider(config.apiKey, config.model);
101
+ case "anthropic":
102
+ return new AnthropicProvider(config.apiKey, config.model);
103
+ default:
104
+ throw new Error(`Unknown provider: ${config.provider}`);
105
+ }
106
+ }
@@ -0,0 +1,10 @@
1
+ /**
2
+ * Client-side PII redaction using regex patterns.
3
+ * Runs entirely locally — no data leaves before redaction.
4
+ * Ported from crowdlisten_deployed/frontend/src/lib/piiRedactor.js
5
+ */
6
+ import type { RedactionResult } from "./types.js";
7
+ /**
8
+ * Redact PII from text using local regex patterns.
9
+ */
10
+ export declare function redactPII(text: string): RedactionResult;
@@ -0,0 +1,68 @@
1
+ /**
2
+ * Client-side PII redaction using regex patterns.
3
+ * Runs entirely locally — no data leaves before redaction.
4
+ * Ported from crowdlisten_deployed/frontend/src/lib/piiRedactor.js
5
+ */
6
+ const PATTERNS = [
7
+ {
8
+ name: "emails",
9
+ regex: /[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}/g,
10
+ replacement: "[EMAIL]",
11
+ },
12
+ {
13
+ name: "phones",
14
+ regex: /(?:\+?\d{1,3}[-.\s]?)?\(?\d{2,4}\)?[-.\s]?\d{3,4}[-.\s]?\d{3,4}\b/g,
15
+ replacement: "[PHONE]",
16
+ },
17
+ {
18
+ name: "ssns",
19
+ regex: /\b\d{3}-\d{2}-\d{4}\b/g,
20
+ replacement: "[SSN]",
21
+ },
22
+ {
23
+ name: "creditCards",
24
+ regex: /\b(?:\d[ -]*?){13,19}\b/g,
25
+ replacement: "[CREDIT_CARD]",
26
+ },
27
+ {
28
+ name: "ipAddresses",
29
+ regex: /\b(?:\d{1,3}\.){3}\d{1,3}\b/g,
30
+ replacement: "[IP_ADDRESS]",
31
+ },
32
+ {
33
+ name: "apiKeys",
34
+ regex: /\b(?:sk-|pk-|api[_-]|token[_-]|bearer\s+|ghp_|gho_|xox[bpsa]-)[A-Za-z0-9_\-/.]{20,}\b/gi,
35
+ replacement: "[API_KEY]",
36
+ },
37
+ {
38
+ name: "genericTokens",
39
+ regex: /\b[A-Fa-f0-9]{40,}\b/g,
40
+ replacement: "[TOKEN]",
41
+ },
42
+ {
43
+ name: "urlCredentials",
44
+ regex: /https?:\/\/[^:]+:[^@]+@[^\s]+/g,
45
+ replacement: "[URL_WITH_CREDENTIALS]",
46
+ },
47
+ ];
48
+ /**
49
+ * Redact PII from text using local regex patterns.
50
+ */
51
+ export function redactPII(text) {
52
+ if (!text)
53
+ return { redactedText: "", stats: {}, totalRedactions: 0 };
54
+ let redactedText = text;
55
+ const stats = {};
56
+ let totalRedactions = 0;
57
+ for (const { name, regex, replacement } of PATTERNS) {
58
+ regex.lastIndex = 0;
59
+ const matches = redactedText.match(regex);
60
+ const count = matches?.length || 0;
61
+ if (count > 0) {
62
+ stats[name] = count;
63
+ totalRedactions += count;
64
+ redactedText = redactedText.replace(regex, replacement);
65
+ }
66
+ }
67
+ return { redactedText, stats, totalRedactions };
68
+ }
@@ -0,0 +1,5 @@
1
+ /**
2
+ * Express server for context extraction web UI.
3
+ * Serves static files + API routes on port 3847.
4
+ */
5
+ export declare function startContextServer(): Promise<void>;
@@ -0,0 +1,134 @@
1
+ /**
2
+ * Express server for context extraction web UI.
3
+ * Serves static files + API routes on port 3847.
4
+ */
5
+ import * as http from "http";
6
+ import * as path from "path";
7
+ import * as fs from "fs";
8
+ import { fileURLToPath } from "url";
9
+ import { handleApiRequest } from "./api.js";
10
+ const __filename = fileURLToPath(import.meta.url);
11
+ const __dirname = path.dirname(__filename);
12
+ const PORT = 3847;
13
+ // MIME types for static file serving
14
+ const MIME = {
15
+ ".html": "text/html",
16
+ ".js": "application/javascript",
17
+ ".css": "text/css",
18
+ ".json": "application/json",
19
+ ".png": "image/png",
20
+ ".svg": "image/svg+xml",
21
+ ".ico": "image/x-icon",
22
+ };
23
+ function serveStatic(res, filePath) {
24
+ const ext = path.extname(filePath);
25
+ const mime = MIME[ext] || "application/octet-stream";
26
+ try {
27
+ const content = fs.readFileSync(filePath);
28
+ res.writeHead(200, { "Content-Type": mime });
29
+ res.end(content);
30
+ }
31
+ catch {
32
+ // SPA fallback: serve index.html for any non-API, non-file route
33
+ try {
34
+ const indexPath = path.join(getWebDistDir(), "index.html");
35
+ const content = fs.readFileSync(indexPath);
36
+ res.writeHead(200, { "Content-Type": "text/html" });
37
+ res.end(content);
38
+ }
39
+ catch {
40
+ res.writeHead(404);
41
+ res.end("Not found");
42
+ }
43
+ }
44
+ }
45
+ function getWebDistDir() {
46
+ // Check for built web UI in various locations
47
+ const candidates = [
48
+ path.join(__dirname, "..", "..", "web-dist"),
49
+ path.join(__dirname, "..", "web-dist"),
50
+ path.join(__dirname, "web-dist"),
51
+ ];
52
+ for (const dir of candidates) {
53
+ if (fs.existsSync(path.join(dir, "index.html"))) {
54
+ return dir;
55
+ }
56
+ }
57
+ // Fallback: return the first candidate (will show instructions if missing)
58
+ return candidates[0];
59
+ }
60
+ function openBrowser(url) {
61
+ import("child_process").then(({ execSync }) => {
62
+ try {
63
+ if (process.platform === "darwin")
64
+ execSync(`open "${url}"`);
65
+ else if (process.platform === "win32")
66
+ execSync(`start "${url}"`);
67
+ else
68
+ execSync(`xdg-open "${url}" || sensible-browser "${url}"`);
69
+ }
70
+ catch {
71
+ // Silently fail — user can open manually
72
+ }
73
+ });
74
+ }
75
+ export async function startContextServer() {
76
+ const webDir = getWebDistDir();
77
+ const hasWebUI = fs.existsSync(path.join(webDir, "index.html"));
78
+ const server = http.createServer(async (req, res) => {
79
+ const url = new URL(req.url || "/", `http://localhost:${PORT}`);
80
+ // CORS headers for local development
81
+ res.setHeader("Access-Control-Allow-Origin", "*");
82
+ res.setHeader("Access-Control-Allow-Methods", "GET, POST, PUT, DELETE, OPTIONS");
83
+ res.setHeader("Access-Control-Allow-Headers", "Content-Type");
84
+ if (req.method === "OPTIONS") {
85
+ res.writeHead(204);
86
+ res.end();
87
+ return;
88
+ }
89
+ // API routes
90
+ if (url.pathname.startsWith("/api/")) {
91
+ await handleApiRequest(req, res, url);
92
+ return;
93
+ }
94
+ // Static files
95
+ if (hasWebUI) {
96
+ const filePath = path.join(webDir, url.pathname === "/" ? "index.html" : url.pathname);
97
+ serveStatic(res, filePath);
98
+ }
99
+ else {
100
+ // No web UI built — show instructions
101
+ res.writeHead(200, { "Content-Type": "text/html" });
102
+ res.end(`
103
+ <!DOCTYPE html>
104
+ <html>
105
+ <head><title>CrowdListen Context</title></head>
106
+ <body style="font-family: system-ui; max-width: 600px; margin: 60px auto; padding: 0 20px;">
107
+ <h1>CrowdListen Context</h1>
108
+ <p>The web UI hasn't been built yet. You can:</p>
109
+ <ol>
110
+ <li>Use the CLI: <code>npx @crowdlisten/harness context &lt;file&gt;</code></li>
111
+ <li>Build the web UI: <code>npm run build:web</code></li>
112
+ </ol>
113
+ <h2>API Endpoints</h2>
114
+ <ul>
115
+ <li><code>POST /api/process</code> — Process text through the pipeline</li>
116
+ <li><code>GET /api/blocks</code> — Get stored context blocks</li>
117
+ <li><code>GET /api/skills</code> — Get skill recommendations</li>
118
+ <li><code>GET /api/config</code> — Get current config status</li>
119
+ <li><code>POST /api/config</code> — Update LLM config</li>
120
+ </ul>
121
+ </body>
122
+ </html>`);
123
+ }
124
+ });
125
+ server.listen(PORT, "127.0.0.1", () => {
126
+ console.error(`\n🚀 CrowdListen Context server running at http://localhost:${PORT}\n`);
127
+ if (hasWebUI) {
128
+ openBrowser(`http://localhost:${PORT}`);
129
+ }
130
+ else {
131
+ console.error(" Web UI not built. Run 'npm run build:web' or use the API directly.\n");
132
+ }
133
+ });
134
+ }
@@ -0,0 +1,12 @@
1
+ /**
2
+ * Local JSON storage in ~/.crowdlisten/ for context blocks, config, and history.
3
+ */
4
+ import type { ContextBlock, ContextConfig, StoredData } from "./types.js";
5
+ export declare function loadConfig(): ContextConfig | null;
6
+ export declare function saveConfig(config: ContextConfig): void;
7
+ export declare function getBlocks(): ContextBlock[];
8
+ export declare function addBlocks(blocks: ContextBlock[], source: string): ContextBlock[];
9
+ export declare function deleteBlock(index: number): void;
10
+ export declare function updateBlock(index: number, updates: Partial<ContextBlock>): void;
11
+ export declare function clearBlocks(): void;
12
+ export declare function getHistory(): StoredData["history"];
@@ -0,0 +1,82 @@
1
+ /**
2
+ * Local JSON storage in ~/.crowdlisten/ for context blocks, config, and history.
3
+ */
4
+ import * as fs from "fs";
5
+ import * as path from "path";
6
+ import * as os from "os";
7
+ const BASE_DIR = path.join(os.homedir(), ".crowdlisten");
8
+ const CONFIG_FILE = path.join(BASE_DIR, "config.json");
9
+ const DATA_FILE = path.join(BASE_DIR, "context.json");
10
+ function ensureDir() {
11
+ if (!fs.existsSync(BASE_DIR)) {
12
+ fs.mkdirSync(BASE_DIR, { recursive: true });
13
+ }
14
+ }
15
+ // ─── Config ──────────────────────────────────────────────────────────────────
16
+ export function loadConfig() {
17
+ try {
18
+ if (!fs.existsSync(CONFIG_FILE))
19
+ return null;
20
+ const raw = fs.readFileSync(CONFIG_FILE, "utf-8");
21
+ return JSON.parse(raw);
22
+ }
23
+ catch {
24
+ return null;
25
+ }
26
+ }
27
+ export function saveConfig(config) {
28
+ ensureDir();
29
+ fs.writeFileSync(CONFIG_FILE, JSON.stringify(config, null, 2));
30
+ }
31
+ // ─── Context Data ────────────────────────────────────────────────────────────
32
+ function loadData() {
33
+ try {
34
+ if (!fs.existsSync(DATA_FILE))
35
+ return { blocks: [], history: [] };
36
+ const raw = fs.readFileSync(DATA_FILE, "utf-8");
37
+ return JSON.parse(raw);
38
+ }
39
+ catch {
40
+ return { blocks: [], history: [] };
41
+ }
42
+ }
43
+ function saveData(data) {
44
+ ensureDir();
45
+ fs.writeFileSync(DATA_FILE, JSON.stringify(data, null, 2));
46
+ }
47
+ export function getBlocks() {
48
+ return loadData().blocks;
49
+ }
50
+ export function addBlocks(blocks, source) {
51
+ const data = loadData();
52
+ data.blocks.push(...blocks);
53
+ data.history.push({
54
+ timestamp: new Date().toISOString(),
55
+ source,
56
+ blockCount: blocks.length,
57
+ });
58
+ saveData(data);
59
+ return data.blocks;
60
+ }
61
+ export function deleteBlock(index) {
62
+ const data = loadData();
63
+ if (index >= 0 && index < data.blocks.length) {
64
+ data.blocks.splice(index, 1);
65
+ saveData(data);
66
+ }
67
+ }
68
+ export function updateBlock(index, updates) {
69
+ const data = loadData();
70
+ if (index >= 0 && index < data.blocks.length) {
71
+ data.blocks[index] = { ...data.blocks[index], ...updates };
72
+ saveData(data);
73
+ }
74
+ }
75
+ export function clearBlocks() {
76
+ const data = loadData();
77
+ data.blocks = [];
78
+ saveData(data);
79
+ }
80
+ export function getHistory() {
81
+ return loadData().history;
82
+ }
@@ -0,0 +1,79 @@
1
+ /**
2
+ * Type definitions for the context extraction pipeline.
3
+ */
4
+ export interface RedactionResult {
5
+ redactedText: string;
6
+ stats: Record<string, number>;
7
+ totalRedactions: number;
8
+ }
9
+ export interface ContextBlock {
10
+ type: "style" | "insight" | "pattern" | "preference" | "decision";
11
+ title: string;
12
+ content: string;
13
+ source?: string;
14
+ }
15
+ export interface SkillMatch {
16
+ skillId: string;
17
+ name: string;
18
+ description: string;
19
+ score: number;
20
+ matchedKeywords: string[];
21
+ }
22
+ export interface SkillCatalogEntry {
23
+ id: string;
24
+ name: string;
25
+ description: string;
26
+ keywords: string[];
27
+ }
28
+ export type SkillTier = "crowdlisten" | "community";
29
+ export type SkillCategory = "development" | "data" | "content" | "research" | "automation" | "design" | "business" | "productivity";
30
+ export type InstallMethod = "copy" | "npx" | "git-clone";
31
+ export interface ExtendedSkillCatalogEntry extends SkillCatalogEntry {
32
+ tier: SkillTier;
33
+ category: SkillCategory;
34
+ installMethod: InstallMethod;
35
+ installTarget: string;
36
+ source: string;
37
+ author?: string;
38
+ }
39
+ export interface ExtendedSkillMatch extends SkillMatch {
40
+ tier: SkillTier;
41
+ category: SkillCategory;
42
+ installMethod: InstallMethod;
43
+ installTarget: string;
44
+ }
45
+ export interface LLMMessage {
46
+ role: "system" | "user" | "assistant";
47
+ content: string;
48
+ }
49
+ export interface LLMOpts {
50
+ model?: string;
51
+ temperature?: number;
52
+ maxTokens?: number;
53
+ jsonMode?: boolean;
54
+ }
55
+ export interface LLMProvider {
56
+ name: string;
57
+ complete(messages: LLMMessage[], opts?: LLMOpts): Promise<string>;
58
+ embed?(texts: string[]): Promise<number[][]>;
59
+ }
60
+ export interface ContextConfig {
61
+ provider: "openai" | "anthropic";
62
+ apiKey: string;
63
+ model?: string;
64
+ }
65
+ export interface PipelineResult {
66
+ blocks: ContextBlock[];
67
+ skills: SkillMatch[];
68
+ redactionStats: Record<string, number>;
69
+ totalRedactions: number;
70
+ chunkCount: number;
71
+ }
72
+ export interface StoredData {
73
+ blocks: ContextBlock[];
74
+ history: Array<{
75
+ timestamp: string;
76
+ source: string;
77
+ blockCount: number;
78
+ }>;
79
+ }
@@ -0,0 +1,4 @@
1
+ /**
2
+ * Type definitions for the context extraction pipeline.
3
+ */
4
+ export {};