@crowdlisten/harness 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +167 -0
- package/LICENSE +21 -0
- package/README.md +153 -0
- package/dist/agent-proxy.d.ts +24 -0
- package/dist/agent-proxy.js +140 -0
- package/dist/agent-tools.d.ts +736 -0
- package/dist/agent-tools.js +409 -0
- package/dist/context/api.d.ts +5 -0
- package/dist/context/api.js +164 -0
- package/dist/context/cli.d.ts +19 -0
- package/dist/context/cli.js +108 -0
- package/dist/context/extractor.d.ts +12 -0
- package/dist/context/extractor.js +43 -0
- package/dist/context/index.d.ts +12 -0
- package/dist/context/index.js +11 -0
- package/dist/context/matcher.d.ts +39 -0
- package/dist/context/matcher.js +246 -0
- package/dist/context/parser.d.ts +28 -0
- package/dist/context/parser.js +157 -0
- package/dist/context/pipeline.d.ts +26 -0
- package/dist/context/pipeline.js +56 -0
- package/dist/context/prompts.d.ts +6 -0
- package/dist/context/prompts.js +60 -0
- package/dist/context/providers.d.ts +6 -0
- package/dist/context/providers.js +106 -0
- package/dist/context/redactor.d.ts +10 -0
- package/dist/context/redactor.js +68 -0
- package/dist/context/server.d.ts +5 -0
- package/dist/context/server.js +134 -0
- package/dist/context/store.d.ts +12 -0
- package/dist/context/store.js +82 -0
- package/dist/context/types.d.ts +79 -0
- package/dist/context/types.js +4 -0
- package/dist/context/user-state.d.ts +40 -0
- package/dist/context/user-state.js +144 -0
- package/dist/index.d.ts +14 -0
- package/dist/index.js +385 -0
- package/dist/insights/browser/BrowserPool.d.ts +87 -0
- package/dist/insights/browser/BrowserPool.js +266 -0
- package/dist/insights/browser/RequestInterceptor.d.ts +46 -0
- package/dist/insights/browser/RequestInterceptor.js +115 -0
- package/dist/insights/cli.d.ts +8 -0
- package/dist/insights/cli.js +206 -0
- package/dist/insights/core/base/BaseAdapter.d.ts +37 -0
- package/dist/insights/core/base/BaseAdapter.js +123 -0
- package/dist/insights/core/health/HealthMonitor.d.ts +75 -0
- package/dist/insights/core/health/HealthMonitor.js +171 -0
- package/dist/insights/core/interfaces/SocialMediaPlatform.d.ts +125 -0
- package/dist/insights/core/interfaces/SocialMediaPlatform.js +42 -0
- package/dist/insights/core/utils/DataNormalizer.d.ts +53 -0
- package/dist/insights/core/utils/DataNormalizer.js +349 -0
- package/dist/insights/core/utils/InstagramUrlUtils.d.ts +11 -0
- package/dist/insights/core/utils/InstagramUrlUtils.js +60 -0
- package/dist/insights/core/utils/TikTokUrlUtils.d.ts +10 -0
- package/dist/insights/core/utils/TikTokUrlUtils.js +57 -0
- package/dist/insights/handlers.d.ts +157 -0
- package/dist/insights/handlers.js +246 -0
- package/dist/insights/index.d.ts +437 -0
- package/dist/insights/index.js +426 -0
- package/dist/insights/platforms/instagram/InstagramAdapter.d.ts +34 -0
- package/dist/insights/platforms/instagram/InstagramAdapter.js +342 -0
- package/dist/insights/platforms/moltbook/MoltbookAdapter.d.ts +31 -0
- package/dist/insights/platforms/moltbook/MoltbookAdapter.js +227 -0
- package/dist/insights/platforms/reddit/RedditAdapter.d.ts +21 -0
- package/dist/insights/platforms/reddit/RedditAdapter.js +212 -0
- package/dist/insights/platforms/tiktok/TikTokAdapter.d.ts +34 -0
- package/dist/insights/platforms/tiktok/TikTokAdapter.js +269 -0
- package/dist/insights/platforms/twitter/TwitterAdapter.d.ts +23 -0
- package/dist/insights/platforms/twitter/TwitterAdapter.js +211 -0
- package/dist/insights/platforms/xiaohongshu/XiaohongshuAdapter.d.ts +35 -0
- package/dist/insights/platforms/xiaohongshu/XiaohongshuAdapter.js +258 -0
- package/dist/insights/platforms/youtube/YouTubeAdapter.d.ts +22 -0
- package/dist/insights/platforms/youtube/YouTubeAdapter.js +254 -0
- package/dist/insights/service-config.d.ts +7 -0
- package/dist/insights/service-config.js +60 -0
- package/dist/insights/services/UnifiedSocialMediaService.d.ts +94 -0
- package/dist/insights/services/UnifiedSocialMediaService.js +259 -0
- package/dist/insights/vision/VisionExtractor.d.ts +46 -0
- package/dist/insights/vision/VisionExtractor.js +236 -0
- package/dist/learnings.d.ts +50 -0
- package/dist/learnings.js +130 -0
- package/dist/openapi.d.ts +29 -0
- package/dist/openapi.js +169 -0
- package/dist/server-factory.d.ts +20 -0
- package/dist/server-factory.js +41 -0
- package/dist/suggestions.d.ts +16 -0
- package/dist/suggestions.js +72 -0
- package/dist/telemetry.d.ts +44 -0
- package/dist/telemetry.js +93 -0
- package/dist/tools/registry.d.ts +65 -0
- package/dist/tools/registry.js +256 -0
- package/dist/tools.d.ts +2433 -0
- package/dist/tools.js +2294 -0
- package/dist/transport/http.d.ts +15 -0
- package/dist/transport/http.js +154 -0
- package/package.json +76 -0
- package/skills/catalog.json +272 -0
- package/skills/community-catalog.json +4202 -0
- package/skills/competitive-analysis/SKILL.md +174 -0
- package/skills/content-creator/SKILL.md +256 -0
- package/skills/content-strategy/SKILL.md +222 -0
- package/skills/data-storytelling/SKILL.md +248 -0
- package/skills/heuristic-evaluation/SKILL.md +201 -0
- package/skills/market-research-reports/SKILL.md +184 -0
- package/skills/user-stories/SKILL.md +178 -0
- package/skills/ux-researcher/SKILL.md +239 -0
- package/web-dist/assets/index-B1b25lNd.css +1 -0
- package/web-dist/assets/index-CDWHwHbl.js +64 -0
- package/web-dist/index.html +16 -0
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Orchestrator: parse -> redact -> chunk -> extract -> match -> store
|
|
3
|
+
*/
|
|
4
|
+
import { redactPII } from "./redactor.js";
|
|
5
|
+
import { parseFile, chunkText } from "./parser.js";
|
|
6
|
+
import { extractBlocks } from "./extractor.js";
|
|
7
|
+
import { matchSkills } from "./matcher.js";
|
|
8
|
+
import { createProvider } from "./providers.js";
|
|
9
|
+
import { addBlocks, loadConfig } from "./store.js";
|
|
10
|
+
/**
|
|
11
|
+
* Run the full context extraction pipeline.
|
|
12
|
+
*/
|
|
13
|
+
export async function runPipeline(opts) {
|
|
14
|
+
const config = opts.config || loadConfig();
|
|
15
|
+
if (!config) {
|
|
16
|
+
throw new Error("No LLM provider configured. Run: npx @crowdlisten/harness setup");
|
|
17
|
+
}
|
|
18
|
+
const provider = createProvider(config);
|
|
19
|
+
const source = opts.source || opts.filePath || "paste";
|
|
20
|
+
// 1. Get raw text
|
|
21
|
+
let rawText = opts.text || "";
|
|
22
|
+
if (opts.filePath) {
|
|
23
|
+
const parsed = await parseFile(opts.filePath);
|
|
24
|
+
rawText = rawText ? rawText + "\n\n---\n\n" + parsed : parsed;
|
|
25
|
+
}
|
|
26
|
+
if (!rawText.trim()) {
|
|
27
|
+
throw new Error("No content to process");
|
|
28
|
+
}
|
|
29
|
+
// 2. PII redaction
|
|
30
|
+
const { redactedText, stats: redactionStats, totalRedactions } = redactPII(rawText);
|
|
31
|
+
// 3. Chunk
|
|
32
|
+
const chunks = chunkText(redactedText);
|
|
33
|
+
const total = chunks.length;
|
|
34
|
+
// 4. Extract blocks from each chunk
|
|
35
|
+
const allBlocks = [];
|
|
36
|
+
const isChat = opts.isChat !== false; // default true
|
|
37
|
+
for (let i = 0; i < chunks.length; i++) {
|
|
38
|
+
opts.onProgress?.({ current: i + 1, total, blocksFound: allBlocks.length });
|
|
39
|
+
const blocks = await extractBlocks(provider, chunks[i], source, isChat);
|
|
40
|
+
allBlocks.push(...blocks);
|
|
41
|
+
}
|
|
42
|
+
// 5. Match skills
|
|
43
|
+
const skills = await matchSkills(allBlocks);
|
|
44
|
+
// 6. Store locally
|
|
45
|
+
if (allBlocks.length > 0) {
|
|
46
|
+
addBlocks(allBlocks, source);
|
|
47
|
+
}
|
|
48
|
+
opts.onProgress?.({ current: total, total, blocksFound: allBlocks.length });
|
|
49
|
+
return {
|
|
50
|
+
blocks: allBlocks,
|
|
51
|
+
skills,
|
|
52
|
+
redactionStats,
|
|
53
|
+
totalRedactions,
|
|
54
|
+
chunkCount: total,
|
|
55
|
+
};
|
|
56
|
+
}
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* System prompts for LLM-based context extraction.
|
|
3
|
+
* Extracted from crowdlisten_deployed/frontend/src/services/contextBlockService.js
|
|
4
|
+
*/
|
|
5
|
+
export declare const EXTRACT_SYSTEM_PROMPT = "You are a context extraction engine. Analyze the provided text and extract reusable context blocks.\n\nExtract two categories:\n1. **style** \u2014 Writing patterns, tone, communication style, formatting habits, voice characteristics\n2. **insight** \u2014 Key facts, quotes, data points, actionable knowledge, domain expertise\n\nReturn JSON with this exact schema:\n{\n \"blocks\": [\n {\n \"type\": \"style\" | \"insight\",\n \"title\": \"Short descriptive title (3-8 words)\",\n \"content\": \"The extracted context block content (1-3 sentences)\"\n }\n ]\n}\n\nRules:\n- Extract 3-10 blocks depending on content richness\n- Each block should be self-contained and reusable\n- Style blocks capture HOW the person communicates\n- Insight blocks capture WHAT knowledge is present\n- Titles should be specific and descriptive\n- Content should be concise but complete\n- Return valid JSON only, no markdown fences";
|
|
6
|
+
export declare const CHAT_EXTRACT_SYSTEM_PROMPT = "You are a context extraction engine analyzing AI chat history. Extract reusable context blocks that capture the user's patterns, knowledge, and preferences.\n\nExtract four categories:\n1. **style** \u2014 Writing patterns, tone, communication style, formatting habits, voice characteristics\n2. **insight** \u2014 Key facts, domain expertise, actionable knowledge, important decisions or conclusions\n3. **pattern** \u2014 Recurring workflows, decision-making approaches, problem-solving methods, repeated processes\n4. **preference** \u2014 Tool choices, format preferences, working style, technology opinions, environment setup\n\nReturn JSON with this exact schema:\n{\n \"blocks\": [\n {\n \"type\": \"style\" | \"insight\" | \"pattern\" | \"preference\",\n \"title\": \"Short descriptive title (3-8 words)\",\n \"content\": \"The extracted context block content (1-3 sentences)\"\n }\n ]\n}\n\nRules:\n- Extract 5-15 blocks depending on content richness\n- Each block should be self-contained and reusable across conversations\n- Deduplicate \u2014 if the same theme appears multiple times, synthesize into one block\n- Focus on the USER's behavior, not the assistant's\n- Style blocks capture HOW the person communicates\n- Insight blocks capture WHAT knowledge or expertise they have\n- Pattern blocks capture HOW they work and make decisions\n- Preference blocks capture WHAT tools/formats/approaches they choose\n- Titles should be specific and descriptive\n- Content should be concise but complete\n- Return valid JSON only, no markdown fences";
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* System prompts for LLM-based context extraction.
|
|
3
|
+
* Extracted from crowdlisten_deployed/frontend/src/services/contextBlockService.js
|
|
4
|
+
*/
|
|
5
|
+
export const EXTRACT_SYSTEM_PROMPT = `You are a context extraction engine. Analyze the provided text and extract reusable context blocks.
|
|
6
|
+
|
|
7
|
+
Extract two categories:
|
|
8
|
+
1. **style** — Writing patterns, tone, communication style, formatting habits, voice characteristics
|
|
9
|
+
2. **insight** — Key facts, quotes, data points, actionable knowledge, domain expertise
|
|
10
|
+
|
|
11
|
+
Return JSON with this exact schema:
|
|
12
|
+
{
|
|
13
|
+
"blocks": [
|
|
14
|
+
{
|
|
15
|
+
"type": "style" | "insight",
|
|
16
|
+
"title": "Short descriptive title (3-8 words)",
|
|
17
|
+
"content": "The extracted context block content (1-3 sentences)"
|
|
18
|
+
}
|
|
19
|
+
]
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
Rules:
|
|
23
|
+
- Extract 3-10 blocks depending on content richness
|
|
24
|
+
- Each block should be self-contained and reusable
|
|
25
|
+
- Style blocks capture HOW the person communicates
|
|
26
|
+
- Insight blocks capture WHAT knowledge is present
|
|
27
|
+
- Titles should be specific and descriptive
|
|
28
|
+
- Content should be concise but complete
|
|
29
|
+
- Return valid JSON only, no markdown fences`;
|
|
30
|
+
export const CHAT_EXTRACT_SYSTEM_PROMPT = `You are a context extraction engine analyzing AI chat history. Extract reusable context blocks that capture the user's patterns, knowledge, and preferences.
|
|
31
|
+
|
|
32
|
+
Extract four categories:
|
|
33
|
+
1. **style** — Writing patterns, tone, communication style, formatting habits, voice characteristics
|
|
34
|
+
2. **insight** — Key facts, domain expertise, actionable knowledge, important decisions or conclusions
|
|
35
|
+
3. **pattern** — Recurring workflows, decision-making approaches, problem-solving methods, repeated processes
|
|
36
|
+
4. **preference** — Tool choices, format preferences, working style, technology opinions, environment setup
|
|
37
|
+
|
|
38
|
+
Return JSON with this exact schema:
|
|
39
|
+
{
|
|
40
|
+
"blocks": [
|
|
41
|
+
{
|
|
42
|
+
"type": "style" | "insight" | "pattern" | "preference",
|
|
43
|
+
"title": "Short descriptive title (3-8 words)",
|
|
44
|
+
"content": "The extracted context block content (1-3 sentences)"
|
|
45
|
+
}
|
|
46
|
+
]
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
Rules:
|
|
50
|
+
- Extract 5-15 blocks depending on content richness
|
|
51
|
+
- Each block should be self-contained and reusable across conversations
|
|
52
|
+
- Deduplicate — if the same theme appears multiple times, synthesize into one block
|
|
53
|
+
- Focus on the USER's behavior, not the assistant's
|
|
54
|
+
- Style blocks capture HOW the person communicates
|
|
55
|
+
- Insight blocks capture WHAT knowledge or expertise they have
|
|
56
|
+
- Pattern blocks capture HOW they work and make decisions
|
|
57
|
+
- Preference blocks capture WHAT tools/formats/approaches they choose
|
|
58
|
+
- Titles should be specific and descriptive
|
|
59
|
+
- Content should be concise but complete
|
|
60
|
+
- Return valid JSON only, no markdown fences`;
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* LLM provider abstraction: OpenAI, Anthropic, Ollama.
|
|
3
|
+
* Uses the user's own API key — configured via `npx @crowdlisten/harness setup`.
|
|
4
|
+
*/
|
|
5
|
+
import type { LLMProvider, ContextConfig } from "./types.js";
|
|
6
|
+
export declare function createProvider(config: ContextConfig): LLMProvider;
|
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* LLM provider abstraction: OpenAI, Anthropic, Ollama.
|
|
3
|
+
* Uses the user's own API key — configured via `npx @crowdlisten/harness setup`.
|
|
4
|
+
*/
|
|
5
|
+
// ─── OpenAI ──────────────────────────────────────────────────────────────────
|
|
6
|
+
class OpenAIProvider {
|
|
7
|
+
name = "openai";
|
|
8
|
+
apiKey;
|
|
9
|
+
defaultModel;
|
|
10
|
+
constructor(apiKey, model) {
|
|
11
|
+
this.apiKey = apiKey;
|
|
12
|
+
this.defaultModel = model || "gpt-4o-mini";
|
|
13
|
+
}
|
|
14
|
+
async complete(messages, opts) {
|
|
15
|
+
const body = {
|
|
16
|
+
model: opts?.model || this.defaultModel,
|
|
17
|
+
messages,
|
|
18
|
+
temperature: opts?.temperature ?? 0.3,
|
|
19
|
+
};
|
|
20
|
+
if (opts?.maxTokens)
|
|
21
|
+
body.max_tokens = opts.maxTokens;
|
|
22
|
+
if (opts?.jsonMode)
|
|
23
|
+
body.response_format = { type: "json_object" };
|
|
24
|
+
const res = await fetch("https://api.openai.com/v1/chat/completions", {
|
|
25
|
+
method: "POST",
|
|
26
|
+
headers: {
|
|
27
|
+
"Content-Type": "application/json",
|
|
28
|
+
Authorization: `Bearer ${this.apiKey}`,
|
|
29
|
+
},
|
|
30
|
+
body: JSON.stringify(body),
|
|
31
|
+
});
|
|
32
|
+
if (!res.ok) {
|
|
33
|
+
const err = await res.json().catch(() => ({}));
|
|
34
|
+
throw new Error(err.error?.message || `OpenAI request failed (${res.status})`);
|
|
35
|
+
}
|
|
36
|
+
const data = (await res.json());
|
|
37
|
+
return data.choices?.[0]?.message?.content || "";
|
|
38
|
+
}
|
|
39
|
+
async embed(texts) {
|
|
40
|
+
const res = await fetch("https://api.openai.com/v1/embeddings", {
|
|
41
|
+
method: "POST",
|
|
42
|
+
headers: {
|
|
43
|
+
"Content-Type": "application/json",
|
|
44
|
+
Authorization: `Bearer ${this.apiKey}`,
|
|
45
|
+
},
|
|
46
|
+
body: JSON.stringify({
|
|
47
|
+
model: "text-embedding-3-small",
|
|
48
|
+
input: texts,
|
|
49
|
+
}),
|
|
50
|
+
});
|
|
51
|
+
if (!res.ok) {
|
|
52
|
+
throw new Error(`OpenAI embeddings failed (${res.status})`);
|
|
53
|
+
}
|
|
54
|
+
const data = (await res.json());
|
|
55
|
+
return data.data.map((d) => d.embedding);
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
// ─── Anthropic ───────────────────────────────────────────────────────────────
|
|
59
|
+
class AnthropicProvider {
|
|
60
|
+
name = "anthropic";
|
|
61
|
+
apiKey;
|
|
62
|
+
defaultModel;
|
|
63
|
+
constructor(apiKey, model) {
|
|
64
|
+
this.apiKey = apiKey;
|
|
65
|
+
this.defaultModel = model || "claude-sonnet-4-20250514";
|
|
66
|
+
}
|
|
67
|
+
async complete(messages, opts) {
|
|
68
|
+
const systemMsg = messages.find((m) => m.role === "system");
|
|
69
|
+
const nonSystem = messages.filter((m) => m.role !== "system");
|
|
70
|
+
const body = {
|
|
71
|
+
model: opts?.model || this.defaultModel,
|
|
72
|
+
max_tokens: opts?.maxTokens || 4096,
|
|
73
|
+
messages: nonSystem.map((m) => ({ role: m.role, content: m.content })),
|
|
74
|
+
};
|
|
75
|
+
if (systemMsg)
|
|
76
|
+
body.system = systemMsg.content;
|
|
77
|
+
const res = await fetch("https://api.anthropic.com/v1/messages", {
|
|
78
|
+
method: "POST",
|
|
79
|
+
headers: {
|
|
80
|
+
"Content-Type": "application/json",
|
|
81
|
+
"x-api-key": this.apiKey,
|
|
82
|
+
"anthropic-version": "2023-06-01",
|
|
83
|
+
},
|
|
84
|
+
body: JSON.stringify(body),
|
|
85
|
+
});
|
|
86
|
+
if (!res.ok) {
|
|
87
|
+
const err = await res.json().catch(() => ({}));
|
|
88
|
+
throw new Error(err.error?.message ||
|
|
89
|
+
`Anthropic request failed (${res.status})`);
|
|
90
|
+
}
|
|
91
|
+
const data = (await res.json());
|
|
92
|
+
const textBlock = data.content?.find((c) => c.type === "text");
|
|
93
|
+
return textBlock?.text || "";
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
// ─── Factory ─────────────────────────────────────────────────────────────────
|
|
97
|
+
export function createProvider(config) {
|
|
98
|
+
switch (config.provider) {
|
|
99
|
+
case "openai":
|
|
100
|
+
return new OpenAIProvider(config.apiKey, config.model);
|
|
101
|
+
case "anthropic":
|
|
102
|
+
return new AnthropicProvider(config.apiKey, config.model);
|
|
103
|
+
default:
|
|
104
|
+
throw new Error(`Unknown provider: ${config.provider}`);
|
|
105
|
+
}
|
|
106
|
+
}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Client-side PII redaction using regex patterns.
|
|
3
|
+
* Runs entirely locally — no data leaves before redaction.
|
|
4
|
+
* Ported from crowdlisten_deployed/frontend/src/lib/piiRedactor.js
|
|
5
|
+
*/
|
|
6
|
+
import type { RedactionResult } from "./types.js";
|
|
7
|
+
/**
|
|
8
|
+
* Redact PII from text using local regex patterns.
|
|
9
|
+
*/
|
|
10
|
+
export declare function redactPII(text: string): RedactionResult;
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Client-side PII redaction using regex patterns.
|
|
3
|
+
* Runs entirely locally — no data leaves before redaction.
|
|
4
|
+
* Ported from crowdlisten_deployed/frontend/src/lib/piiRedactor.js
|
|
5
|
+
*/
|
|
6
|
+
const PATTERNS = [
|
|
7
|
+
{
|
|
8
|
+
name: "emails",
|
|
9
|
+
regex: /[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}/g,
|
|
10
|
+
replacement: "[EMAIL]",
|
|
11
|
+
},
|
|
12
|
+
{
|
|
13
|
+
name: "phones",
|
|
14
|
+
regex: /(?:\+?\d{1,3}[-.\s]?)?\(?\d{2,4}\)?[-.\s]?\d{3,4}[-.\s]?\d{3,4}\b/g,
|
|
15
|
+
replacement: "[PHONE]",
|
|
16
|
+
},
|
|
17
|
+
{
|
|
18
|
+
name: "ssns",
|
|
19
|
+
regex: /\b\d{3}-\d{2}-\d{4}\b/g,
|
|
20
|
+
replacement: "[SSN]",
|
|
21
|
+
},
|
|
22
|
+
{
|
|
23
|
+
name: "creditCards",
|
|
24
|
+
regex: /\b(?:\d[ -]*?){13,19}\b/g,
|
|
25
|
+
replacement: "[CREDIT_CARD]",
|
|
26
|
+
},
|
|
27
|
+
{
|
|
28
|
+
name: "ipAddresses",
|
|
29
|
+
regex: /\b(?:\d{1,3}\.){3}\d{1,3}\b/g,
|
|
30
|
+
replacement: "[IP_ADDRESS]",
|
|
31
|
+
},
|
|
32
|
+
{
|
|
33
|
+
name: "apiKeys",
|
|
34
|
+
regex: /\b(?:sk-|pk-|api[_-]|token[_-]|bearer\s+|ghp_|gho_|xox[bpsa]-)[A-Za-z0-9_\-/.]{20,}\b/gi,
|
|
35
|
+
replacement: "[API_KEY]",
|
|
36
|
+
},
|
|
37
|
+
{
|
|
38
|
+
name: "genericTokens",
|
|
39
|
+
regex: /\b[A-Fa-f0-9]{40,}\b/g,
|
|
40
|
+
replacement: "[TOKEN]",
|
|
41
|
+
},
|
|
42
|
+
{
|
|
43
|
+
name: "urlCredentials",
|
|
44
|
+
regex: /https?:\/\/[^:]+:[^@]+@[^\s]+/g,
|
|
45
|
+
replacement: "[URL_WITH_CREDENTIALS]",
|
|
46
|
+
},
|
|
47
|
+
];
|
|
48
|
+
/**
|
|
49
|
+
* Redact PII from text using local regex patterns.
|
|
50
|
+
*/
|
|
51
|
+
export function redactPII(text) {
|
|
52
|
+
if (!text)
|
|
53
|
+
return { redactedText: "", stats: {}, totalRedactions: 0 };
|
|
54
|
+
let redactedText = text;
|
|
55
|
+
const stats = {};
|
|
56
|
+
let totalRedactions = 0;
|
|
57
|
+
for (const { name, regex, replacement } of PATTERNS) {
|
|
58
|
+
regex.lastIndex = 0;
|
|
59
|
+
const matches = redactedText.match(regex);
|
|
60
|
+
const count = matches?.length || 0;
|
|
61
|
+
if (count > 0) {
|
|
62
|
+
stats[name] = count;
|
|
63
|
+
totalRedactions += count;
|
|
64
|
+
redactedText = redactedText.replace(regex, replacement);
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
return { redactedText, stats, totalRedactions };
|
|
68
|
+
}
|
|
@@ -0,0 +1,134 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Express server for context extraction web UI.
|
|
3
|
+
* Serves static files + API routes on port 3847.
|
|
4
|
+
*/
|
|
5
|
+
import * as http from "http";
|
|
6
|
+
import * as path from "path";
|
|
7
|
+
import * as fs from "fs";
|
|
8
|
+
import { fileURLToPath } from "url";
|
|
9
|
+
import { handleApiRequest } from "./api.js";
|
|
10
|
+
const __filename = fileURLToPath(import.meta.url);
|
|
11
|
+
const __dirname = path.dirname(__filename);
|
|
12
|
+
const PORT = 3847;
|
|
13
|
+
// MIME types for static file serving
|
|
14
|
+
const MIME = {
|
|
15
|
+
".html": "text/html",
|
|
16
|
+
".js": "application/javascript",
|
|
17
|
+
".css": "text/css",
|
|
18
|
+
".json": "application/json",
|
|
19
|
+
".png": "image/png",
|
|
20
|
+
".svg": "image/svg+xml",
|
|
21
|
+
".ico": "image/x-icon",
|
|
22
|
+
};
|
|
23
|
+
function serveStatic(res, filePath) {
|
|
24
|
+
const ext = path.extname(filePath);
|
|
25
|
+
const mime = MIME[ext] || "application/octet-stream";
|
|
26
|
+
try {
|
|
27
|
+
const content = fs.readFileSync(filePath);
|
|
28
|
+
res.writeHead(200, { "Content-Type": mime });
|
|
29
|
+
res.end(content);
|
|
30
|
+
}
|
|
31
|
+
catch {
|
|
32
|
+
// SPA fallback: serve index.html for any non-API, non-file route
|
|
33
|
+
try {
|
|
34
|
+
const indexPath = path.join(getWebDistDir(), "index.html");
|
|
35
|
+
const content = fs.readFileSync(indexPath);
|
|
36
|
+
res.writeHead(200, { "Content-Type": "text/html" });
|
|
37
|
+
res.end(content);
|
|
38
|
+
}
|
|
39
|
+
catch {
|
|
40
|
+
res.writeHead(404);
|
|
41
|
+
res.end("Not found");
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
function getWebDistDir() {
|
|
46
|
+
// Check for built web UI in various locations
|
|
47
|
+
const candidates = [
|
|
48
|
+
path.join(__dirname, "..", "..", "web-dist"),
|
|
49
|
+
path.join(__dirname, "..", "web-dist"),
|
|
50
|
+
path.join(__dirname, "web-dist"),
|
|
51
|
+
];
|
|
52
|
+
for (const dir of candidates) {
|
|
53
|
+
if (fs.existsSync(path.join(dir, "index.html"))) {
|
|
54
|
+
return dir;
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
// Fallback: return the first candidate (will show instructions if missing)
|
|
58
|
+
return candidates[0];
|
|
59
|
+
}
|
|
60
|
+
function openBrowser(url) {
|
|
61
|
+
import("child_process").then(({ execSync }) => {
|
|
62
|
+
try {
|
|
63
|
+
if (process.platform === "darwin")
|
|
64
|
+
execSync(`open "${url}"`);
|
|
65
|
+
else if (process.platform === "win32")
|
|
66
|
+
execSync(`start "${url}"`);
|
|
67
|
+
else
|
|
68
|
+
execSync(`xdg-open "${url}" || sensible-browser "${url}"`);
|
|
69
|
+
}
|
|
70
|
+
catch {
|
|
71
|
+
// Silently fail — user can open manually
|
|
72
|
+
}
|
|
73
|
+
});
|
|
74
|
+
}
|
|
75
|
+
export async function startContextServer() {
|
|
76
|
+
const webDir = getWebDistDir();
|
|
77
|
+
const hasWebUI = fs.existsSync(path.join(webDir, "index.html"));
|
|
78
|
+
const server = http.createServer(async (req, res) => {
|
|
79
|
+
const url = new URL(req.url || "/", `http://localhost:${PORT}`);
|
|
80
|
+
// CORS headers for local development
|
|
81
|
+
res.setHeader("Access-Control-Allow-Origin", "*");
|
|
82
|
+
res.setHeader("Access-Control-Allow-Methods", "GET, POST, PUT, DELETE, OPTIONS");
|
|
83
|
+
res.setHeader("Access-Control-Allow-Headers", "Content-Type");
|
|
84
|
+
if (req.method === "OPTIONS") {
|
|
85
|
+
res.writeHead(204);
|
|
86
|
+
res.end();
|
|
87
|
+
return;
|
|
88
|
+
}
|
|
89
|
+
// API routes
|
|
90
|
+
if (url.pathname.startsWith("/api/")) {
|
|
91
|
+
await handleApiRequest(req, res, url);
|
|
92
|
+
return;
|
|
93
|
+
}
|
|
94
|
+
// Static files
|
|
95
|
+
if (hasWebUI) {
|
|
96
|
+
const filePath = path.join(webDir, url.pathname === "/" ? "index.html" : url.pathname);
|
|
97
|
+
serveStatic(res, filePath);
|
|
98
|
+
}
|
|
99
|
+
else {
|
|
100
|
+
// No web UI built — show instructions
|
|
101
|
+
res.writeHead(200, { "Content-Type": "text/html" });
|
|
102
|
+
res.end(`
|
|
103
|
+
<!DOCTYPE html>
|
|
104
|
+
<html>
|
|
105
|
+
<head><title>CrowdListen Context</title></head>
|
|
106
|
+
<body style="font-family: system-ui; max-width: 600px; margin: 60px auto; padding: 0 20px;">
|
|
107
|
+
<h1>CrowdListen Context</h1>
|
|
108
|
+
<p>The web UI hasn't been built yet. You can:</p>
|
|
109
|
+
<ol>
|
|
110
|
+
<li>Use the CLI: <code>npx @crowdlisten/harness context <file></code></li>
|
|
111
|
+
<li>Build the web UI: <code>npm run build:web</code></li>
|
|
112
|
+
</ol>
|
|
113
|
+
<h2>API Endpoints</h2>
|
|
114
|
+
<ul>
|
|
115
|
+
<li><code>POST /api/process</code> — Process text through the pipeline</li>
|
|
116
|
+
<li><code>GET /api/blocks</code> — Get stored context blocks</li>
|
|
117
|
+
<li><code>GET /api/skills</code> — Get skill recommendations</li>
|
|
118
|
+
<li><code>GET /api/config</code> — Get current config status</li>
|
|
119
|
+
<li><code>POST /api/config</code> — Update LLM config</li>
|
|
120
|
+
</ul>
|
|
121
|
+
</body>
|
|
122
|
+
</html>`);
|
|
123
|
+
}
|
|
124
|
+
});
|
|
125
|
+
server.listen(PORT, "127.0.0.1", () => {
|
|
126
|
+
console.error(`\n🚀 CrowdListen Context server running at http://localhost:${PORT}\n`);
|
|
127
|
+
if (hasWebUI) {
|
|
128
|
+
openBrowser(`http://localhost:${PORT}`);
|
|
129
|
+
}
|
|
130
|
+
else {
|
|
131
|
+
console.error(" Web UI not built. Run 'npm run build:web' or use the API directly.\n");
|
|
132
|
+
}
|
|
133
|
+
});
|
|
134
|
+
}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Local JSON storage in ~/.crowdlisten/ for context blocks, config, and history.
|
|
3
|
+
*/
|
|
4
|
+
import type { ContextBlock, ContextConfig, StoredData } from "./types.js";
|
|
5
|
+
export declare function loadConfig(): ContextConfig | null;
|
|
6
|
+
export declare function saveConfig(config: ContextConfig): void;
|
|
7
|
+
export declare function getBlocks(): ContextBlock[];
|
|
8
|
+
export declare function addBlocks(blocks: ContextBlock[], source: string): ContextBlock[];
|
|
9
|
+
export declare function deleteBlock(index: number): void;
|
|
10
|
+
export declare function updateBlock(index: number, updates: Partial<ContextBlock>): void;
|
|
11
|
+
export declare function clearBlocks(): void;
|
|
12
|
+
export declare function getHistory(): StoredData["history"];
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Local JSON storage in ~/.crowdlisten/ for context blocks, config, and history.
|
|
3
|
+
*/
|
|
4
|
+
import * as fs from "fs";
|
|
5
|
+
import * as path from "path";
|
|
6
|
+
import * as os from "os";
|
|
7
|
+
const BASE_DIR = path.join(os.homedir(), ".crowdlisten");
|
|
8
|
+
const CONFIG_FILE = path.join(BASE_DIR, "config.json");
|
|
9
|
+
const DATA_FILE = path.join(BASE_DIR, "context.json");
|
|
10
|
+
function ensureDir() {
|
|
11
|
+
if (!fs.existsSync(BASE_DIR)) {
|
|
12
|
+
fs.mkdirSync(BASE_DIR, { recursive: true });
|
|
13
|
+
}
|
|
14
|
+
}
|
|
15
|
+
// ─── Config ──────────────────────────────────────────────────────────────────
|
|
16
|
+
export function loadConfig() {
|
|
17
|
+
try {
|
|
18
|
+
if (!fs.existsSync(CONFIG_FILE))
|
|
19
|
+
return null;
|
|
20
|
+
const raw = fs.readFileSync(CONFIG_FILE, "utf-8");
|
|
21
|
+
return JSON.parse(raw);
|
|
22
|
+
}
|
|
23
|
+
catch {
|
|
24
|
+
return null;
|
|
25
|
+
}
|
|
26
|
+
}
|
|
27
|
+
export function saveConfig(config) {
|
|
28
|
+
ensureDir();
|
|
29
|
+
fs.writeFileSync(CONFIG_FILE, JSON.stringify(config, null, 2));
|
|
30
|
+
}
|
|
31
|
+
// ─── Context Data ────────────────────────────────────────────────────────────
|
|
32
|
+
function loadData() {
|
|
33
|
+
try {
|
|
34
|
+
if (!fs.existsSync(DATA_FILE))
|
|
35
|
+
return { blocks: [], history: [] };
|
|
36
|
+
const raw = fs.readFileSync(DATA_FILE, "utf-8");
|
|
37
|
+
return JSON.parse(raw);
|
|
38
|
+
}
|
|
39
|
+
catch {
|
|
40
|
+
return { blocks: [], history: [] };
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
function saveData(data) {
|
|
44
|
+
ensureDir();
|
|
45
|
+
fs.writeFileSync(DATA_FILE, JSON.stringify(data, null, 2));
|
|
46
|
+
}
|
|
47
|
+
export function getBlocks() {
|
|
48
|
+
return loadData().blocks;
|
|
49
|
+
}
|
|
50
|
+
export function addBlocks(blocks, source) {
|
|
51
|
+
const data = loadData();
|
|
52
|
+
data.blocks.push(...blocks);
|
|
53
|
+
data.history.push({
|
|
54
|
+
timestamp: new Date().toISOString(),
|
|
55
|
+
source,
|
|
56
|
+
blockCount: blocks.length,
|
|
57
|
+
});
|
|
58
|
+
saveData(data);
|
|
59
|
+
return data.blocks;
|
|
60
|
+
}
|
|
61
|
+
export function deleteBlock(index) {
|
|
62
|
+
const data = loadData();
|
|
63
|
+
if (index >= 0 && index < data.blocks.length) {
|
|
64
|
+
data.blocks.splice(index, 1);
|
|
65
|
+
saveData(data);
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
export function updateBlock(index, updates) {
|
|
69
|
+
const data = loadData();
|
|
70
|
+
if (index >= 0 && index < data.blocks.length) {
|
|
71
|
+
data.blocks[index] = { ...data.blocks[index], ...updates };
|
|
72
|
+
saveData(data);
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
export function clearBlocks() {
|
|
76
|
+
const data = loadData();
|
|
77
|
+
data.blocks = [];
|
|
78
|
+
saveData(data);
|
|
79
|
+
}
|
|
80
|
+
export function getHistory() {
|
|
81
|
+
return loadData().history;
|
|
82
|
+
}
|
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Type definitions for the context extraction pipeline.
|
|
3
|
+
*/
|
|
4
|
+
export interface RedactionResult {
|
|
5
|
+
redactedText: string;
|
|
6
|
+
stats: Record<string, number>;
|
|
7
|
+
totalRedactions: number;
|
|
8
|
+
}
|
|
9
|
+
export interface ContextBlock {
|
|
10
|
+
type: "style" | "insight" | "pattern" | "preference" | "decision";
|
|
11
|
+
title: string;
|
|
12
|
+
content: string;
|
|
13
|
+
source?: string;
|
|
14
|
+
}
|
|
15
|
+
export interface SkillMatch {
|
|
16
|
+
skillId: string;
|
|
17
|
+
name: string;
|
|
18
|
+
description: string;
|
|
19
|
+
score: number;
|
|
20
|
+
matchedKeywords: string[];
|
|
21
|
+
}
|
|
22
|
+
export interface SkillCatalogEntry {
|
|
23
|
+
id: string;
|
|
24
|
+
name: string;
|
|
25
|
+
description: string;
|
|
26
|
+
keywords: string[];
|
|
27
|
+
}
|
|
28
|
+
export type SkillTier = "crowdlisten" | "community";
|
|
29
|
+
export type SkillCategory = "development" | "data" | "content" | "research" | "automation" | "design" | "business" | "productivity";
|
|
30
|
+
export type InstallMethod = "copy" | "npx" | "git-clone";
|
|
31
|
+
export interface ExtendedSkillCatalogEntry extends SkillCatalogEntry {
|
|
32
|
+
tier: SkillTier;
|
|
33
|
+
category: SkillCategory;
|
|
34
|
+
installMethod: InstallMethod;
|
|
35
|
+
installTarget: string;
|
|
36
|
+
source: string;
|
|
37
|
+
author?: string;
|
|
38
|
+
}
|
|
39
|
+
export interface ExtendedSkillMatch extends SkillMatch {
|
|
40
|
+
tier: SkillTier;
|
|
41
|
+
category: SkillCategory;
|
|
42
|
+
installMethod: InstallMethod;
|
|
43
|
+
installTarget: string;
|
|
44
|
+
}
|
|
45
|
+
export interface LLMMessage {
|
|
46
|
+
role: "system" | "user" | "assistant";
|
|
47
|
+
content: string;
|
|
48
|
+
}
|
|
49
|
+
export interface LLMOpts {
|
|
50
|
+
model?: string;
|
|
51
|
+
temperature?: number;
|
|
52
|
+
maxTokens?: number;
|
|
53
|
+
jsonMode?: boolean;
|
|
54
|
+
}
|
|
55
|
+
export interface LLMProvider {
|
|
56
|
+
name: string;
|
|
57
|
+
complete(messages: LLMMessage[], opts?: LLMOpts): Promise<string>;
|
|
58
|
+
embed?(texts: string[]): Promise<number[][]>;
|
|
59
|
+
}
|
|
60
|
+
export interface ContextConfig {
|
|
61
|
+
provider: "openai" | "anthropic";
|
|
62
|
+
apiKey: string;
|
|
63
|
+
model?: string;
|
|
64
|
+
}
|
|
65
|
+
export interface PipelineResult {
|
|
66
|
+
blocks: ContextBlock[];
|
|
67
|
+
skills: SkillMatch[];
|
|
68
|
+
redactionStats: Record<string, number>;
|
|
69
|
+
totalRedactions: number;
|
|
70
|
+
chunkCount: number;
|
|
71
|
+
}
|
|
72
|
+
export interface StoredData {
|
|
73
|
+
blocks: ContextBlock[];
|
|
74
|
+
history: Array<{
|
|
75
|
+
timestamp: string;
|
|
76
|
+
source: string;
|
|
77
|
+
blockCount: number;
|
|
78
|
+
}>;
|
|
79
|
+
}
|