context-mem 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.context-mem.json.example +22 -0
- package/LICENSE +21 -0
- package/README.md +213 -0
- package/dist/cli/commands/dashboard.d.ts +2 -0
- package/dist/cli/commands/dashboard.d.ts.map +1 -0
- package/dist/cli/commands/dashboard.js +55 -0
- package/dist/cli/commands/dashboard.js.map +1 -0
- package/dist/cli/commands/doctor.d.ts +2 -0
- package/dist/cli/commands/doctor.d.ts.map +1 -0
- package/dist/cli/commands/doctor.js +49 -0
- package/dist/cli/commands/doctor.js.map +1 -0
- package/dist/cli/commands/init.d.ts +2 -0
- package/dist/cli/commands/init.d.ts.map +1 -0
- package/dist/cli/commands/init.js +43 -0
- package/dist/cli/commands/init.js.map +1 -0
- package/dist/cli/commands/serve.d.ts +2 -0
- package/dist/cli/commands/serve.d.ts.map +1 -0
- package/dist/cli/commands/serve.js +47 -0
- package/dist/cli/commands/serve.js.map +1 -0
- package/dist/cli/commands/status.d.ts +2 -0
- package/dist/cli/commands/status.d.ts.map +1 -0
- package/dist/cli/commands/status.js +29 -0
- package/dist/cli/commands/status.js.map +1 -0
- package/dist/cli/index.d.ts +3 -0
- package/dist/cli/index.d.ts.map +1 -0
- package/dist/cli/index.js +39 -0
- package/dist/cli/index.js.map +1 -0
- package/dist/core/budget.d.ts +11 -0
- package/dist/core/budget.d.ts.map +1 -0
- package/dist/core/budget.js +60 -0
- package/dist/core/budget.js.map +1 -0
- package/dist/core/config.d.ts +4 -0
- package/dist/core/config.d.ts.map +1 -0
- package/dist/core/config.js +41 -0
- package/dist/core/config.js.map +1 -0
- package/dist/core/events.d.ts +19 -0
- package/dist/core/events.d.ts.map +1 -0
- package/dist/core/events.js +101 -0
- package/dist/core/events.js.map +1 -0
- package/dist/core/kernel.d.ts +46 -0
- package/dist/core/kernel.d.ts.map +1 -0
- package/dist/core/kernel.js +228 -0
- package/dist/core/kernel.js.map +1 -0
- package/dist/core/lifecycle.d.ts +19 -0
- package/dist/core/lifecycle.d.ts.map +1 -0
- package/dist/core/lifecycle.js +35 -0
- package/dist/core/lifecycle.js.map +1 -0
- package/dist/core/observe-queue.d.ts +18 -0
- package/dist/core/observe-queue.d.ts.map +1 -0
- package/dist/core/observe-queue.js +56 -0
- package/dist/core/observe-queue.js.map +1 -0
- package/dist/core/pipeline.d.ts +15 -0
- package/dist/core/pipeline.d.ts.map +1 -0
- package/dist/core/pipeline.js +114 -0
- package/dist/core/pipeline.js.map +1 -0
- package/dist/core/plugin-registry.d.ts +13 -0
- package/dist/core/plugin-registry.d.ts.map +1 -0
- package/dist/core/plugin-registry.js +67 -0
- package/dist/core/plugin-registry.js.map +1 -0
- package/dist/core/session.d.ts +14 -0
- package/dist/core/session.d.ts.map +1 -0
- package/dist/core/session.js +95 -0
- package/dist/core/session.js.map +1 -0
- package/dist/core/truncation.d.ts +11 -0
- package/dist/core/truncation.d.ts.map +1 -0
- package/dist/core/truncation.js +143 -0
- package/dist/core/truncation.js.map +1 -0
- package/dist/core/types.d.ts +230 -0
- package/dist/core/types.d.ts.map +1 -0
- package/dist/core/types.js +50 -0
- package/dist/core/types.js.map +1 -0
- package/dist/core/utils.d.ts +4 -0
- package/dist/core/utils.d.ts.map +1 -0
- package/dist/core/utils.js +57 -0
- package/dist/core/utils.js.map +1 -0
- package/dist/index.d.ts +12 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +27 -0
- package/dist/index.js.map +1 -0
- package/dist/mcp-server/server.d.ts +5 -0
- package/dist/mcp-server/server.d.ts.map +1 -0
- package/dist/mcp-server/server.js +101 -0
- package/dist/mcp-server/server.js.map +1 -0
- package/dist/mcp-server/tools.d.ts +201 -0
- package/dist/mcp-server/tools.d.ts.map +1 -0
- package/dist/mcp-server/tools.js +618 -0
- package/dist/mcp-server/tools.js.map +1 -0
- package/dist/plugins/knowledge/knowledge-base.d.ts +23 -0
- package/dist/plugins/knowledge/knowledge-base.d.ts.map +1 -0
- package/dist/plugins/knowledge/knowledge-base.js +165 -0
- package/dist/plugins/knowledge/knowledge-base.js.map +1 -0
- package/dist/plugins/platforms/claude-code.d.ts +14 -0
- package/dist/plugins/platforms/claude-code.d.ts.map +1 -0
- package/dist/plugins/platforms/claude-code.js +52 -0
- package/dist/plugins/platforms/claude-code.js.map +1 -0
- package/dist/plugins/privacy/privacy-engine.d.ts +18 -0
- package/dist/plugins/privacy/privacy-engine.d.ts.map +1 -0
- package/dist/plugins/privacy/privacy-engine.js +72 -0
- package/dist/plugins/privacy/privacy-engine.js.map +1 -0
- package/dist/plugins/runtimes/javascript.d.ts +13 -0
- package/dist/plugins/runtimes/javascript.d.ts.map +1 -0
- package/dist/plugins/runtimes/javascript.js +102 -0
- package/dist/plugins/runtimes/javascript.js.map +1 -0
- package/dist/plugins/runtimes/python.d.ts +14 -0
- package/dist/plugins/runtimes/python.d.ts.map +1 -0
- package/dist/plugins/runtimes/python.js +127 -0
- package/dist/plugins/runtimes/python.js.map +1 -0
- package/dist/plugins/runtimes/shell.d.ts +13 -0
- package/dist/plugins/runtimes/shell.d.ts.map +1 -0
- package/dist/plugins/runtimes/shell.js +55 -0
- package/dist/plugins/runtimes/shell.js.map +1 -0
- package/dist/plugins/search/bm25.d.ts +16 -0
- package/dist/plugins/search/bm25.d.ts.map +1 -0
- package/dist/plugins/search/bm25.js +61 -0
- package/dist/plugins/search/bm25.js.map +1 -0
- package/dist/plugins/search/fts5-utils.d.ts +2 -0
- package/dist/plugins/search/fts5-utils.d.ts.map +1 -0
- package/dist/plugins/search/fts5-utils.js +12 -0
- package/dist/plugins/search/fts5-utils.js.map +1 -0
- package/dist/plugins/search/fusion.d.ts +9 -0
- package/dist/plugins/search/fusion.d.ts.map +1 -0
- package/dist/plugins/search/fusion.js +45 -0
- package/dist/plugins/search/fusion.js.map +1 -0
- package/dist/plugins/search/intent.d.ts +5 -0
- package/dist/plugins/search/intent.d.ts.map +1 -0
- package/dist/plugins/search/intent.js +49 -0
- package/dist/plugins/search/intent.js.map +1 -0
- package/dist/plugins/search/levenshtein.d.ts +17 -0
- package/dist/plugins/search/levenshtein.d.ts.map +1 -0
- package/dist/plugins/search/levenshtein.js +99 -0
- package/dist/plugins/search/levenshtein.js.map +1 -0
- package/dist/plugins/search/trigram.d.ts +16 -0
- package/dist/plugins/search/trigram.d.ts.map +1 -0
- package/dist/plugins/search/trigram.js +63 -0
- package/dist/plugins/search/trigram.js.map +1 -0
- package/dist/plugins/storage/better-sqlite3.d.ts +19 -0
- package/dist/plugins/storage/better-sqlite3.d.ts.map +1 -0
- package/dist/plugins/storage/better-sqlite3.js +82 -0
- package/dist/plugins/storage/better-sqlite3.js.map +1 -0
- package/dist/plugins/storage/content-store.d.ts +20 -0
- package/dist/plugins/storage/content-store.d.ts.map +1 -0
- package/dist/plugins/storage/content-store.js +187 -0
- package/dist/plugins/storage/content-store.js.map +1 -0
- package/dist/plugins/storage/migrations.d.ts +8 -0
- package/dist/plugins/storage/migrations.d.ts.map +1 -0
- package/dist/plugins/storage/migrations.js +252 -0
- package/dist/plugins/storage/migrations.js.map +1 -0
- package/dist/plugins/summarizers/binary-summarizer.d.ts +12 -0
- package/dist/plugins/summarizers/binary-summarizer.d.ts.map +1 -0
- package/dist/plugins/summarizers/binary-summarizer.js +43 -0
- package/dist/plugins/summarizers/binary-summarizer.js.map +1 -0
- package/dist/plugins/summarizers/build-output-summarizer.d.ts +12 -0
- package/dist/plugins/summarizers/build-output-summarizer.d.ts.map +1 -0
- package/dist/plugins/summarizers/build-output-summarizer.js +68 -0
- package/dist/plugins/summarizers/build-output-summarizer.js.map +1 -0
- package/dist/plugins/summarizers/code-summarizer.d.ts +12 -0
- package/dist/plugins/summarizers/code-summarizer.d.ts.map +1 -0
- package/dist/plugins/summarizers/code-summarizer.js +179 -0
- package/dist/plugins/summarizers/code-summarizer.js.map +1 -0
- package/dist/plugins/summarizers/csv-summarizer.d.ts +12 -0
- package/dist/plugins/summarizers/csv-summarizer.d.ts.map +1 -0
- package/dist/plugins/summarizers/csv-summarizer.js +60 -0
- package/dist/plugins/summarizers/csv-summarizer.js.map +1 -0
- package/dist/plugins/summarizers/error-summarizer.d.ts +15 -0
- package/dist/plugins/summarizers/error-summarizer.d.ts.map +1 -0
- package/dist/plugins/summarizers/error-summarizer.js +111 -0
- package/dist/plugins/summarizers/error-summarizer.js.map +1 -0
- package/dist/plugins/summarizers/git-log-summarizer.d.ts +12 -0
- package/dist/plugins/summarizers/git-log-summarizer.d.ts.map +1 -0
- package/dist/plugins/summarizers/git-log-summarizer.js +76 -0
- package/dist/plugins/summarizers/git-log-summarizer.js.map +1 -0
- package/dist/plugins/summarizers/html-summarizer.d.ts +12 -0
- package/dist/plugins/summarizers/html-summarizer.d.ts.map +1 -0
- package/dist/plugins/summarizers/html-summarizer.js +69 -0
- package/dist/plugins/summarizers/html-summarizer.js.map +1 -0
- package/dist/plugins/summarizers/json-summarizer.d.ts +12 -0
- package/dist/plugins/summarizers/json-summarizer.d.ts.map +1 -0
- package/dist/plugins/summarizers/json-summarizer.js +132 -0
- package/dist/plugins/summarizers/json-summarizer.js.map +1 -0
- package/dist/plugins/summarizers/log-summarizer.d.ts +12 -0
- package/dist/plugins/summarizers/log-summarizer.d.ts.map +1 -0
- package/dist/plugins/summarizers/log-summarizer.js +173 -0
- package/dist/plugins/summarizers/log-summarizer.js.map +1 -0
- package/dist/plugins/summarizers/markdown-summarizer.d.ts +12 -0
- package/dist/plugins/summarizers/markdown-summarizer.d.ts.map +1 -0
- package/dist/plugins/summarizers/markdown-summarizer.js +75 -0
- package/dist/plugins/summarizers/markdown-summarizer.js.map +1 -0
- package/dist/plugins/summarizers/network-summarizer.d.ts +12 -0
- package/dist/plugins/summarizers/network-summarizer.d.ts.map +1 -0
- package/dist/plugins/summarizers/network-summarizer.js +74 -0
- package/dist/plugins/summarizers/network-summarizer.js.map +1 -0
- package/dist/plugins/summarizers/shell-summarizer.d.ts +12 -0
- package/dist/plugins/summarizers/shell-summarizer.d.ts.map +1 -0
- package/dist/plugins/summarizers/shell-summarizer.js +50 -0
- package/dist/plugins/summarizers/shell-summarizer.js.map +1 -0
- package/dist/plugins/summarizers/test-output-summarizer.d.ts +12 -0
- package/dist/plugins/summarizers/test-output-summarizer.d.ts.map +1 -0
- package/dist/plugins/summarizers/test-output-summarizer.js +77 -0
- package/dist/plugins/summarizers/test-output-summarizer.js.map +1 -0
- package/dist/plugins/summarizers/typescript-error-summarizer.d.ts +12 -0
- package/dist/plugins/summarizers/typescript-error-summarizer.d.ts.map +1 -0
- package/dist/plugins/summarizers/typescript-error-summarizer.js +67 -0
- package/dist/plugins/summarizers/typescript-error-summarizer.js.map +1 -0
- package/hooks/context-mem-hook.js +77 -0
- package/hooks/dashboard-autostart.js +92 -0
- package/hooks/dashboard-stop.js +32 -0
- package/hooks/hooks.json +40 -0
- package/package.json +32 -0
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.HtmlSummarizer = void 0;
|
|
4
|
+
const utils_js_1 = require("../../core/utils.js");
|
|
5
|
+
const HTML_DETECT_PATTERN = /<html|<head|<body|<!DOCTYPE/i;
|
|
6
|
+
const HEADING_TAG_REGEX = /<h([1-6])[^>]*>(.*?)<\/h\1>/gi;
|
|
7
|
+
const TITLE_REGEX = /<title[^>]*>(.*?)<\/title>/i;
|
|
8
|
+
const NAV_REGEX = /<nav[^>]*>([\s\S]*?)<\/nav>/gi;
|
|
9
|
+
const FORM_REGEX = /<form[\s>]/gi;
|
|
10
|
+
class HtmlSummarizer {
|
|
11
|
+
name = 'html-summarizer';
|
|
12
|
+
version = '1.0.0';
|
|
13
|
+
type = 'summarizer';
|
|
14
|
+
contentTypes = ['html'];
|
|
15
|
+
async init(_config) { }
|
|
16
|
+
async destroy() { }
|
|
17
|
+
detect(content) {
|
|
18
|
+
return HTML_DETECT_PATTERN.test(content);
|
|
19
|
+
}
|
|
20
|
+
async summarize(content, _opts) {
|
|
21
|
+
const tokensOriginal = (0, utils_js_1.estimateTokens)(content);
|
|
22
|
+
// Extract <title>
|
|
23
|
+
const titleMatch = content.match(TITLE_REGEX);
|
|
24
|
+
const title = titleMatch ? titleMatch[1].trim() : '(no title)';
|
|
25
|
+
// Extract headings
|
|
26
|
+
const headings = [];
|
|
27
|
+
let headingMatch;
|
|
28
|
+
const headingRegex = new RegExp(HEADING_TAG_REGEX.source, 'gi');
|
|
29
|
+
while ((headingMatch = headingRegex.exec(content)) !== null) {
|
|
30
|
+
const level = headingMatch[1];
|
|
31
|
+
const text = headingMatch[2].replace(/<[^>]*>/g, '').trim();
|
|
32
|
+
headings.push(`h${level}: ${text}`);
|
|
33
|
+
}
|
|
34
|
+
// Find <nav> sections
|
|
35
|
+
const navSections = [];
|
|
36
|
+
let navMatch;
|
|
37
|
+
const navRegex = new RegExp(NAV_REGEX.source, 'gi');
|
|
38
|
+
while ((navMatch = navRegex.exec(content)) !== null) {
|
|
39
|
+
const navText = navMatch[1].replace(/<[^>]*>/g, ' ').replace(/\s+/g, ' ').trim();
|
|
40
|
+
navSections.push(navText.slice(0, 200));
|
|
41
|
+
}
|
|
42
|
+
// Count <form> elements
|
|
43
|
+
const formMatches = content.match(FORM_REGEX);
|
|
44
|
+
const formCount = formMatches ? formMatches.length : 0;
|
|
45
|
+
const summaryParts = [
|
|
46
|
+
`# HTML Document: ${title}`,
|
|
47
|
+
'',
|
|
48
|
+
`## Headings (${headings.length})`,
|
|
49
|
+
...headings.map(h => ` - ${h}`),
|
|
50
|
+
'',
|
|
51
|
+
`## Navigation sections: ${navSections.length}`,
|
|
52
|
+
...navSections.map((n, i) => ` nav[${i}]: ${n}`),
|
|
53
|
+
'',
|
|
54
|
+
`## Forms: ${formCount}`,
|
|
55
|
+
`## Size: ${content.length} chars`,
|
|
56
|
+
];
|
|
57
|
+
const summary = summaryParts.join('\n');
|
|
58
|
+
const tokensSummarized = (0, utils_js_1.estimateTokens)(summary);
|
|
59
|
+
return {
|
|
60
|
+
summary,
|
|
61
|
+
tokens_original: tokensOriginal,
|
|
62
|
+
tokens_summarized: tokensSummarized,
|
|
63
|
+
savings_pct: tokensOriginal > 0 ? Math.round((1 - tokensSummarized / tokensOriginal) * 100) : 0,
|
|
64
|
+
content_type: 'html',
|
|
65
|
+
};
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
exports.HtmlSummarizer = HtmlSummarizer;
|
|
69
|
+
//# sourceMappingURL=html-summarizer.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"html-summarizer.js","sourceRoot":"","sources":["../../../src/plugins/summarizers/html-summarizer.ts"],"names":[],"mappings":";;;AACA,kDAAqD;AAErD,MAAM,mBAAmB,GAAG,8BAA8B,CAAC;AAC3D,MAAM,iBAAiB,GAAG,+BAA+B,CAAC;AAC1D,MAAM,WAAW,GAAG,6BAA6B,CAAC;AAClD,MAAM,SAAS,GAAG,+BAA+B,CAAC;AAClD,MAAM,UAAU,GAAG,cAAc,CAAC;AAElC,MAAa,cAAc;IACzB,IAAI,GAAG,iBAAiB,CAAC;IACzB,OAAO,GAAG,OAAO,CAAC;IAClB,IAAI,GAAG,YAAqB,CAAC;IAC7B,YAAY,GAAG,CAAC,MAAM,CAAC,CAAC;IAExB,KAAK,CAAC,IAAI,CAAC,OAAqB,IAAkB,CAAC;IACnD,KAAK,CAAC,OAAO,KAAmB,CAAC;IAEjC,MAAM,CAAC,OAAe;QACpB,OAAO,mBAAmB,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IAC3C,CAAC;IAED,KAAK,CAAC,SAAS,CAAC,OAAe,EAAE,KAAoB;QACnD,MAAM,cAAc,GAAG,IAAA,yBAAc,EAAC,OAAO,CAAC,CAAC;QAE/C,kBAAkB;QAClB,MAAM,UAAU,GAAG,OAAO,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;QAC9C,MAAM,KAAK,GAAG,UAAU,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,YAAY,CAAC;QAE/D,mBAAmB;QACnB,MAAM,QAAQ,GAAa,EAAE,CAAC;QAC9B,IAAI,YAAY,CAAC;QACjB,MAAM,YAAY,GAAG,IAAI,MAAM,CAAC,iBAAiB,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;QAChE,OAAO,CAAC,YAAY,GAAG,YAAY,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;YAC5D,MAAM,KAAK,GAAG,YAAY,CAAC,CAAC,CAAC,CAAC;YAC9B,MAAM,IAAI,GAAG,YAAY,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;YAC5D,QAAQ,CAAC,IAAI,CAAC,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC,CAAC;QACtC,CAAC;QAED,sBAAsB;QACtB,MAAM,WAAW,GAAa,EAAE,CAAC;QACjC,IAAI,QAAQ,CAAC;QACb,MAAM,QAAQ,GAAG,IAAI,MAAM,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;QACpD,OAAO,CAAC,QAAQ,GAAG,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;YACpD,MAAM,OAAO,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,UAAU,EAAE,GAAG,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;YACjF,WAAW,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC;QAC1C,CAAC;QAED,wBAAwB;QACxB,MAAM,WAAW,GAAG,OAAO,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;QAC9C,MAAM,SAAS,GAAG,WAAW,CAAC,CAAC,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;QAEvD,MAAM,YAAY,GAAG;YACnB,oBAAoB,KAAK,EAAE;YAC3B,EAAE;YACF,gBAAgB,QAAQ,CAAC,MAAM,GAAG;YAClC,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,EAAE,CAAC;YAChC,EAAE;YACF,2BAA2B,WAAW,CAAC,MAAM,EAAE;YAC/C,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC;YACjD,EAAE;YACF,aAAa,SAAS,EAAE;YACxB,YAAY,OAAO,CAAC,MAAM,QAAQ;SACnC,CAAC;QAEF,MAAM,OAAO,GAAG,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACxC,MAAM,gBAAgB,GAAG,IAAA,yBAAc,EAAC,OAAO,CAAC,CAAC;QAEjD,OAAO;YACL,OAAO;YACP,eAAe,EAAE,cAAc;YAC/B,iBAAiB,EAAE,gBAAgB;YACnC,WAAW,EAAE,cAAc,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,gBAAgB,GAAG,cAAc,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;YAC/F,YAAY,EAAE,MAAM;SACrB,CAAC;IACJ,CAAC;CACF;AAnED,wCAmEC"}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import type { SummarizerPlugin, PluginConfig, SummaryResult, SummarizeOpts } from '../../core/types.js';
|
|
2
|
+
export declare class JsonSummarizer implements SummarizerPlugin {
|
|
3
|
+
name: string;
|
|
4
|
+
version: string;
|
|
5
|
+
type: "summarizer";
|
|
6
|
+
contentTypes: string[];
|
|
7
|
+
init(_config: PluginConfig): Promise<void>;
|
|
8
|
+
destroy(): Promise<void>;
|
|
9
|
+
detect(content: string): boolean;
|
|
10
|
+
summarize(content: string, _opts: SummarizeOpts): Promise<SummaryResult>;
|
|
11
|
+
}
|
|
12
|
+
//# sourceMappingURL=json-summarizer.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"json-summarizer.d.ts","sourceRoot":"","sources":["../../../src/plugins/summarizers/json-summarizer.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,gBAAgB,EAAE,YAAY,EAAE,aAAa,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AA8DxG,qBAAa,cAAe,YAAW,gBAAgB;IACrD,IAAI,SAAqB;IACzB,OAAO,SAAW;IAClB,IAAI,EAAG,YAAY,CAAU;IAC7B,YAAY,WAAgC;IAEtC,IAAI,CAAC,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC;IAC1C,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;IAE9B,MAAM,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO;IAW1B,SAAS,CAAC,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,aAAa,GAAG,OAAO,CAAC,aAAa,CAAC;CA4C/E"}
|
|
@@ -0,0 +1,132 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.JsonSummarizer = void 0;
|
|
4
|
+
const utils_js_1 = require("../../core/utils.js");
|
|
5
|
+
const MAX_DEPTH = 3;
|
|
6
|
+
function extractSchema(value, depth, indent) {
|
|
7
|
+
if (depth > MAX_DEPTH)
|
|
8
|
+
return 'object';
|
|
9
|
+
if (Array.isArray(value)) {
|
|
10
|
+
const length = value.length;
|
|
11
|
+
if (length === 0)
|
|
12
|
+
return 'array[0]';
|
|
13
|
+
const firstSchema = extractSchema(value[0], depth + 1, indent + ' ');
|
|
14
|
+
if (typeof value[0] === 'object' && value[0] !== null && !Array.isArray(value[0])) {
|
|
15
|
+
const inner = buildObjectSchema(value[0], depth + 1, indent + ' ');
|
|
16
|
+
return `array[${length}]\n${inner}`;
|
|
17
|
+
}
|
|
18
|
+
return `array[${length}] of ${firstSchema}`;
|
|
19
|
+
}
|
|
20
|
+
if (value === null)
|
|
21
|
+
return 'null';
|
|
22
|
+
if (typeof value === 'object') {
|
|
23
|
+
if (depth >= MAX_DEPTH)
|
|
24
|
+
return 'object';
|
|
25
|
+
return `object\n${buildObjectSchema(value, depth + 1, indent + ' ')}`;
|
|
26
|
+
}
|
|
27
|
+
return typeof value;
|
|
28
|
+
}
|
|
29
|
+
function buildObjectSchema(obj, depth, indent) {
|
|
30
|
+
if (depth > MAX_DEPTH)
|
|
31
|
+
return `${indent}...`;
|
|
32
|
+
const lines = [];
|
|
33
|
+
for (const key of Object.keys(obj)) {
|
|
34
|
+
const val = obj[key];
|
|
35
|
+
if (Array.isArray(val)) {
|
|
36
|
+
const length = val.length;
|
|
37
|
+
if (length === 0) {
|
|
38
|
+
lines.push(`${indent}${key}: array[0]`);
|
|
39
|
+
}
|
|
40
|
+
else if (typeof val[0] === 'object' && val[0] !== null && !Array.isArray(val[0])) {
|
|
41
|
+
lines.push(`${indent}${key}: array[${length}]`);
|
|
42
|
+
if (depth < MAX_DEPTH) {
|
|
43
|
+
lines.push(buildObjectSchema(val[0], depth + 1, indent + ' '));
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
else {
|
|
47
|
+
const firstType = val[0] === null ? 'null' : typeof val[0];
|
|
48
|
+
lines.push(`${indent}${key}: array[${length}] of ${firstType}`);
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
else if (val !== null && typeof val === 'object') {
|
|
52
|
+
if (depth < MAX_DEPTH) {
|
|
53
|
+
lines.push(`${indent}${key}: object`);
|
|
54
|
+
lines.push(buildObjectSchema(val, depth + 1, indent + ' '));
|
|
55
|
+
}
|
|
56
|
+
else {
|
|
57
|
+
lines.push(`${indent}${key}: object`);
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
else {
|
|
61
|
+
const typeName = val === null ? 'null' : typeof val;
|
|
62
|
+
lines.push(`${indent}${key}: ${typeName}`);
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
return lines.join('\n');
|
|
66
|
+
}
|
|
67
|
+
class JsonSummarizer {
|
|
68
|
+
name = 'json-summarizer';
|
|
69
|
+
version = '1.0.0';
|
|
70
|
+
type = 'summarizer';
|
|
71
|
+
contentTypes = ['json', 'application/json'];
|
|
72
|
+
async init(_config) { }
|
|
73
|
+
async destroy() { }
|
|
74
|
+
detect(content) {
|
|
75
|
+
const trimmed = content.trim();
|
|
76
|
+
if (!trimmed.startsWith('{') && !trimmed.startsWith('['))
|
|
77
|
+
return false;
|
|
78
|
+
try {
|
|
79
|
+
JSON.parse(trimmed);
|
|
80
|
+
return true;
|
|
81
|
+
}
|
|
82
|
+
catch {
|
|
83
|
+
return false;
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
async summarize(content, _opts) {
|
|
87
|
+
const tokensOriginal = (0, utils_js_1.estimateTokens)(content);
|
|
88
|
+
let parsed;
|
|
89
|
+
try {
|
|
90
|
+
parsed = JSON.parse(content);
|
|
91
|
+
}
|
|
92
|
+
catch {
|
|
93
|
+
return {
|
|
94
|
+
summary: content,
|
|
95
|
+
tokens_original: tokensOriginal,
|
|
96
|
+
tokens_summarized: tokensOriginal,
|
|
97
|
+
savings_pct: 0,
|
|
98
|
+
content_type: 'json',
|
|
99
|
+
};
|
|
100
|
+
}
|
|
101
|
+
let summary;
|
|
102
|
+
if (Array.isArray(parsed)) {
|
|
103
|
+
const length = parsed.length;
|
|
104
|
+
if (length === 0) {
|
|
105
|
+
summary = `array[0]`;
|
|
106
|
+
}
|
|
107
|
+
else if (typeof parsed[0] === 'object' && parsed[0] !== null && !Array.isArray(parsed[0])) {
|
|
108
|
+
summary = `array[${length}]\n` + buildObjectSchema(parsed[0], 1, ' ');
|
|
109
|
+
}
|
|
110
|
+
else {
|
|
111
|
+
const firstType = parsed[0] === null ? 'null' : typeof parsed[0];
|
|
112
|
+
summary = `array[${length}] of ${firstType}`;
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
else if (parsed !== null && typeof parsed === 'object') {
|
|
116
|
+
summary = buildObjectSchema(parsed, 1, ' ');
|
|
117
|
+
}
|
|
118
|
+
else {
|
|
119
|
+
summary = typeof parsed;
|
|
120
|
+
}
|
|
121
|
+
const tokensSummarized = (0, utils_js_1.estimateTokens)(summary);
|
|
122
|
+
return {
|
|
123
|
+
summary,
|
|
124
|
+
tokens_original: tokensOriginal,
|
|
125
|
+
tokens_summarized: tokensSummarized,
|
|
126
|
+
savings_pct: tokensOriginal > 0 ? Math.round((1 - tokensSummarized / tokensOriginal) * 100) : 0,
|
|
127
|
+
content_type: 'json',
|
|
128
|
+
};
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
exports.JsonSummarizer = JsonSummarizer;
|
|
132
|
+
//# sourceMappingURL=json-summarizer.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"json-summarizer.js","sourceRoot":"","sources":["../../../src/plugins/summarizers/json-summarizer.ts"],"names":[],"mappings":";;;AACA,kDAAqD;AAErD,MAAM,SAAS,GAAG,CAAC,CAAC;AAEpB,SAAS,aAAa,CAAC,KAAc,EAAE,KAAa,EAAE,MAAc;IAClE,IAAI,KAAK,GAAG,SAAS;QAAE,OAAO,QAAQ,CAAC;IAEvC,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;QACzB,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC;QAC5B,IAAI,MAAM,KAAK,CAAC;YAAE,OAAO,UAAU,CAAC;QACpC,MAAM,WAAW,GAAG,aAAa,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,GAAG,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC,CAAC;QACtE,IAAI,OAAO,KAAK,CAAC,CAAC,CAAC,KAAK,QAAQ,IAAI,KAAK,CAAC,CAAC,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;YAClF,MAAM,KAAK,GAAG,iBAAiB,CAAC,KAAK,CAAC,CAAC,CAA4B,EAAE,KAAK,GAAG,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC,CAAC;YAC/F,OAAO,SAAS,MAAM,MAAM,KAAK,EAAE,CAAC;QACtC,CAAC;QACD,OAAO,SAAS,MAAM,QAAQ,WAAW,EAAE,CAAC;IAC9C,CAAC;IAED,IAAI,KAAK,KAAK,IAAI;QAAE,OAAO,MAAM,CAAC;IAElC,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;QAC9B,IAAI,KAAK,IAAI,SAAS;YAAE,OAAO,QAAQ,CAAC;QACxC,OAAO,WAAW,iBAAiB,CAAC,KAAgC,EAAE,KAAK,GAAG,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC,EAAE,CAAC;IACpG,CAAC;IAED,OAAO,OAAO,KAAK,CAAC;AACtB,CAAC;AAED,SAAS,iBAAiB,CAAC,GAA4B,EAAE,KAAa,EAAE,MAAc;IACpF,IAAI,KAAK,GAAG,SAAS;QAAE,OAAO,GAAG,MAAM,KAAK,CAAC;IAC7C,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;QACnC,MAAM,GAAG,GAAG,GAAG,CAAC,GAAG,CAAC,CAAC;QACrB,IAAI,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC;YACvB,MAAM,MAAM,GAAG,GAAG,CAAC,MAAM,CAAC;YAC1B,IAAI,MAAM,KAAK,CAAC,EAAE,CAAC;gBACjB,KAAK,CAAC,IAAI,CAAC,GAAG,MAAM,GAAG,GAAG,YAAY,CAAC,CAAC;YAC1C,CAAC;iBAAM,IAAI,OAAO,GAAG,CAAC,CAAC,CAAC,KAAK,QAAQ,IAAI,GAAG,CAAC,CAAC,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;gBACnF,KAAK,CAAC,IAAI,CAAC,GAAG,MAAM,GAAG,GAAG,WAAW,MAAM,GAAG,CAAC,CAAC;gBAChD,IAAI,KAAK,GAAG,SAAS,EAAE,CAAC;oBACtB,KAAK,CAAC,IAAI,CAAC,iBAAiB,CAAC,GAAG,CAAC,CAAC,CAA4B,EAAE,KAAK,GAAG,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC,CAAC,CAAC;gBAC7F,CAAC;YACH,CAAC;iBAAM,CAAC;gBACN,MAAM,SAAS,GAAG,GAAG,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC;gBAC3D,KAAK,CAAC,IAAI,CAAC,GAAG,MAAM,GAAG,GAAG,WAAW,MAAM,QAAQ,SAAS,EAAE,CAAC,CAAC;YAClE,CAAC;QACH,CAAC;aAAM,IAAI,GAAG,KAAK,IAAI,IAAI,OAAO,GAAG,KAAK,QAAQ,EAAE,CAAC;YACnD,IAAI,KAAK,GAAG,SAAS,EAAE,CAAC;gBACtB,KAAK,CAAC,IAAI,CAAC,GAAG,MAAM,GAAG,GAAG,UAAU,CAAC,CAAC;gBACtC,KAAK,CAAC,IAAI,CAAC,iBAAiB,CAAC,GAA8B,EAAE,KAAK,GAAG,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC,CAAC,CAAC;YAC1F,CAAC;iBAAM,CAAC;gBACN,KAAK,CAAC,IAAI,CAAC,GAAG,MAAM,GAAG,GAAG,UAAU,CAAC,CAAC;YACxC,CAAC;QACH,CAAC;aAAM,CAAC;YACN,MAAM,QAAQ,GAAG,GAAG,KAAK,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,GAAG,CAAC;YACpD,KAAK,CAAC,IAAI,CAAC,GAAG,MAAM,GAAG,GAAG,KAAK,QAAQ,EAAE,CAAC,CAAC;QAC7C,CAAC;IACH,CAAC;IACD,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC;AAED,MAAa,cAAc;IACzB,IAAI,GAAG,iBAAiB,CAAC;IACzB,OAAO,GAAG,OAAO,CAAC;IAClB,IAAI,GAAG,YAAqB,CAAC;IAC7B,YAAY,GAAG,CAAC,MAAM,EAAE,kBAAkB,CAAC,CAAC;IAE5C,KAAK,CAAC,IAAI,CAAC,OAAqB,IAAkB,CAAC;IACnD,KAAK,CAAC,OAAO,KAAmB,CAAC;IAEjC,MAAM,CAAC,OAAe;QACpB,MAAM,OAAO,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC;QAC/B,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC;YAAE,OAAO,KAAK,CAAC;QACvE,IAAI,CAAC;YACH,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;YACpB,OAAO,IAAI,CAAC;QACd,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,KAAK,CAAC;QACf,CAAC;IACH,CAAC;IAED,KAAK,CAAC,SAAS,CAAC,OAAe,EAAE,KAAoB;QACnD,MAAM,cAAc,GAAG,IAAA,yBAAc,EAAC,OAAO,CAAC,CAAC;QAE/C,IAAI,MAAe,CAAC;QACpB,IAAI,CAAC;YACH,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QAC/B,CAAC;QAAC,MAAM,CAAC;YACP,OAAO;gBACL,OAAO,EAAE,OAAO;gBAChB,eAAe,EAAE,cAAc;gBAC/B,iBAAiB,EAAE,cAAc;gBACjC,WAAW,EAAE,CAAC;gBACd,YAAY,EAAE,MAAM;aACrB,CAAC;QACJ,CAAC;QAED,IAAI,OAAe,CAAC;QAEpB,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;YAC1B,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC;YAC7B,IAAI,MAAM,KAAK,CAAC,EAAE,CAAC;gBACjB,OAAO,GAAG,UAAU,CAAC;YACvB,CAAC;iBAAM,IAAI,OAAO,MAAM,CAAC,CAAC,CAAC,KAAK,QAAQ,IAAI,MAAM,CAAC,CAAC,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;gBAC5F,OAAO,GAAG,SAAS,MAAM,KAAK,GAAG,iBAAiB,CAAC,MAAM,CAAC,CAAC,CAA4B,EAAE,CAAC,EAAE,IAAI,CAAC,CAAC;YACpG,CAAC;iBAAM,CAAC;gBACN,MAAM,SAAS,GAAG,MAAM,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,MAAM,CAAC,CAAC,CAAC,CAAC;gBACjE,OAAO,GAAG,SAAS,MAAM,QAAQ,SAAS,EAAE,CAAC;YAC/C,CAAC;QACH,CAAC;aAAM,IAAI,MAAM,KAAK,IAAI,IAAI,OAAO,MAAM,KAAK,QAAQ,EAAE,CAAC;YACzD,OAAO,GAAG,iBAAiB,CAAC,MAAiC,EAAE,CAAC,EAAE,IAAI,CAAC,CAAC;QAC1E,CAAC;aAAM,CAAC;YACN,OAAO,GAAG,OAAO,MAAM,CAAC;QAC1B,CAAC;QAED,MAAM,gBAAgB,GAAG,IAAA,yBAAc,EAAC,OAAO,CAAC,CAAC;QAEjD,OAAO;YACL,OAAO;YACP,eAAe,EAAE,cAAc;YAC/B,iBAAiB,EAAE,gBAAgB;YACnC,WAAW,EAAE,cAAc,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,gBAAgB,GAAG,cAAc,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;YAC/F,YAAY,EAAE,MAAM;SACrB,CAAC;IACJ,CAAC;CACF;AAhED,wCAgEC"}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import type { SummarizerPlugin, PluginConfig, SummaryResult, SummarizeOpts } from '../../core/types.js';
|
|
2
|
+
export declare class LogSummarizer implements SummarizerPlugin {
|
|
3
|
+
name: string;
|
|
4
|
+
version: string;
|
|
5
|
+
type: "summarizer";
|
|
6
|
+
contentTypes: string[];
|
|
7
|
+
init(_config: PluginConfig): Promise<void>;
|
|
8
|
+
destroy(): Promise<void>;
|
|
9
|
+
detect(content: string): boolean;
|
|
10
|
+
summarize(content: string, _opts: SummarizeOpts): Promise<SummaryResult>;
|
|
11
|
+
}
|
|
12
|
+
//# sourceMappingURL=log-summarizer.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"log-summarizer.d.ts","sourceRoot":"","sources":["../../../src/plugins/summarizers/log-summarizer.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,gBAAgB,EAAE,YAAY,EAAE,aAAa,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AA6FxG,qBAAa,aAAc,YAAW,gBAAgB;IACpD,IAAI,SAAoB;IACxB,OAAO,SAAW;IAClB,IAAI,EAAG,YAAY,CAAU;IAC7B,YAAY,WAAsD;IAE5D,IAAI,CAAC,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC;IAC1C,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;IAE9B,MAAM,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO;IAmB1B,SAAS,CAAC,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,aAAa,GAAG,OAAO,CAAC,aAAa,CAAC;CAiE/E"}
|
|
@@ -0,0 +1,173 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.LogSummarizer = void 0;
|
|
4
|
+
const utils_js_1 = require("../../core/utils.js");
|
|
5
|
+
const MIN_LINES_TO_SUMMARIZE = 10;
|
|
6
|
+
// Patterns that indicate log-like content
|
|
7
|
+
const TIMESTAMP_PATTERNS = [
|
|
8
|
+
/\d{4}-\d{2}-\d{2}/, // ISO date: 2024-01-01
|
|
9
|
+
/\d{2}:\d{2}:\d{2}/, // Time: 12:34:56
|
|
10
|
+
];
|
|
11
|
+
const LOG_LEVEL_PATTERNS = [
|
|
12
|
+
/\[(INFO|WARN|ERROR|DEBUG|TRACE|FATAL)\]/, // Bracketed: [INFO]
|
|
13
|
+
/\b(INFO|WARN|ERROR|DEBUG|TRACE|FATAL)\b/, // Bare: INFO
|
|
14
|
+
];
|
|
15
|
+
// Nginx/Apache Combined Log Format detection
|
|
16
|
+
// e.g.: 192.168.1.1 - - [15/Mar/2026:00:00:00 +0000] "GET /path HTTP/1.1" 200 1234 "ref" "ua"
|
|
17
|
+
const ACCESS_LOG_REGEX = /^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\s.*\[.+\]\s"(GET|POST|PUT|DELETE|PATCH|HEAD|OPTIONS)\s+(\S+)\s+HTTP\/[\d.]+"\s+(\d{3})\s+(\d+)/;
|
|
18
|
+
// Strips timestamps from a line for deduplication comparison
|
|
19
|
+
function normalizeLogLine(line) {
|
|
20
|
+
return line
|
|
21
|
+
// ISO datetime with optional milliseconds: 2024-01-01T12:34:56.789Z or 2024-01-01 12:34:56
|
|
22
|
+
.replace(/\d{4}-\d{2}-\d{2}[T ]\d{2}:\d{2}:\d{2}(?:\.\d+)?(?:Z|[+-]\d{2}:\d{2})?/g, '<TIMESTAMP>')
|
|
23
|
+
// Date only: 2024-01-01
|
|
24
|
+
.replace(/\d{4}-\d{2}-\d{2}/g, '<DATE>')
|
|
25
|
+
// Time only: 12:34:56.789
|
|
26
|
+
.replace(/\d{2}:\d{2}:\d{2}(?:\.\d+)?/g, '<TIME>')
|
|
27
|
+
// Unix timestamps (10-13 digit numbers at word boundary)
|
|
28
|
+
.replace(/\b\d{10,13}\b/g, '<TS>')
|
|
29
|
+
.trim();
|
|
30
|
+
}
|
|
31
|
+
function isAccessLog(lines) {
|
|
32
|
+
const sample = lines.slice(0, 20).filter(l => l.trim().length > 0);
|
|
33
|
+
if (sample.length < 5)
|
|
34
|
+
return false;
|
|
35
|
+
let matchCount = 0;
|
|
36
|
+
for (const line of sample) {
|
|
37
|
+
if (ACCESS_LOG_REGEX.test(line))
|
|
38
|
+
matchCount++;
|
|
39
|
+
}
|
|
40
|
+
return matchCount / sample.length >= 0.5;
|
|
41
|
+
}
|
|
42
|
+
function summarizeAccessLog(content) {
|
|
43
|
+
const lines = content.split('\n').filter(l => l.trim().length > 0);
|
|
44
|
+
const methodDist = new Map();
|
|
45
|
+
const statusDist = new Map();
|
|
46
|
+
const endpointDist = new Map();
|
|
47
|
+
let totalBytes = 0;
|
|
48
|
+
for (const line of lines) {
|
|
49
|
+
const m = ACCESS_LOG_REGEX.exec(line);
|
|
50
|
+
if (m) {
|
|
51
|
+
const method = m[1];
|
|
52
|
+
const endpoint = m[2];
|
|
53
|
+
const status = m[3];
|
|
54
|
+
const bytes = parseInt(m[4], 10);
|
|
55
|
+
methodDist.set(method, (methodDist.get(method) || 0) + 1);
|
|
56
|
+
const statusCat = `${status[0]}xx`;
|
|
57
|
+
statusDist.set(statusCat, (statusDist.get(statusCat) || 0) + 1);
|
|
58
|
+
const key = `${method} ${endpoint}`;
|
|
59
|
+
endpointDist.set(key, (endpointDist.get(key) || 0) + 1);
|
|
60
|
+
totalBytes += bytes;
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
const parts = [
|
|
64
|
+
`# Access Log Summary (${lines.length} requests)`,
|
|
65
|
+
'',
|
|
66
|
+
`## Methods`,
|
|
67
|
+
...Array.from(methodDist.entries())
|
|
68
|
+
.sort((a, b) => b[1] - a[1])
|
|
69
|
+
.map(([m, c]) => ` ${m}: ${c}`),
|
|
70
|
+
'',
|
|
71
|
+
`## Status Codes`,
|
|
72
|
+
...Array.from(statusDist.entries())
|
|
73
|
+
.sort()
|
|
74
|
+
.map(([s, c]) => ` ${s}: ${c}`),
|
|
75
|
+
'',
|
|
76
|
+
`## Top Endpoints (${endpointDist.size} unique)`,
|
|
77
|
+
...Array.from(endpointDist.entries())
|
|
78
|
+
.sort((a, b) => b[1] - a[1])
|
|
79
|
+
.slice(0, 20)
|
|
80
|
+
.map(([e, c]) => ` ${e}: ${c}`),
|
|
81
|
+
'',
|
|
82
|
+
`## Transfer: ${(totalBytes / 1024).toFixed(1)} KB total`,
|
|
83
|
+
];
|
|
84
|
+
return parts.join('\n');
|
|
85
|
+
}
|
|
86
|
+
class LogSummarizer {
|
|
87
|
+
name = 'log-summarizer';
|
|
88
|
+
version = '1.0.0';
|
|
89
|
+
type = 'summarizer';
|
|
90
|
+
contentTypes = ['log', 'syslog', 'application-log', 'access-log'];
|
|
91
|
+
async init(_config) { }
|
|
92
|
+
async destroy() { }
|
|
93
|
+
detect(content) {
|
|
94
|
+
const lines = content.split('\n').filter(l => l.trim().length > 0);
|
|
95
|
+
if (lines.length < MIN_LINES_TO_SUMMARIZE)
|
|
96
|
+
return false;
|
|
97
|
+
// Check for access log format first
|
|
98
|
+
if (isAccessLog(lines))
|
|
99
|
+
return true;
|
|
100
|
+
// Check if a meaningful portion of lines match log patterns
|
|
101
|
+
let matchCount = 0;
|
|
102
|
+
for (const line of lines) {
|
|
103
|
+
const hasTimestamp = TIMESTAMP_PATTERNS.some(p => p.test(line));
|
|
104
|
+
const hasLogLevel = LOG_LEVEL_PATTERNS.some(p => p.test(line));
|
|
105
|
+
if (hasTimestamp || hasLogLevel)
|
|
106
|
+
matchCount++;
|
|
107
|
+
}
|
|
108
|
+
// At least 30% of lines should look like log lines
|
|
109
|
+
return matchCount / lines.length >= 0.3;
|
|
110
|
+
}
|
|
111
|
+
async summarize(content, _opts) {
|
|
112
|
+
const lines = content.split('\n');
|
|
113
|
+
const tokensOriginal = (0, utils_js_1.estimateTokens)(content);
|
|
114
|
+
if (lines.length < MIN_LINES_TO_SUMMARIZE) {
|
|
115
|
+
return {
|
|
116
|
+
summary: content,
|
|
117
|
+
tokens_original: tokensOriginal,
|
|
118
|
+
tokens_summarized: tokensOriginal,
|
|
119
|
+
savings_pct: 0,
|
|
120
|
+
content_type: 'log',
|
|
121
|
+
};
|
|
122
|
+
}
|
|
123
|
+
// Access log format gets HTTP-aware aggregation
|
|
124
|
+
const nonEmpty = lines.filter(l => l.trim().length > 0);
|
|
125
|
+
if (isAccessLog(nonEmpty)) {
|
|
126
|
+
const summary = summarizeAccessLog(content);
|
|
127
|
+
const tokensSummarized = (0, utils_js_1.estimateTokens)(summary);
|
|
128
|
+
return {
|
|
129
|
+
summary,
|
|
130
|
+
tokens_original: tokensOriginal,
|
|
131
|
+
tokens_summarized: tokensSummarized,
|
|
132
|
+
savings_pct: tokensOriginal > 0 ? Math.round((1 - tokensSummarized / tokensOriginal) * 100) : 0,
|
|
133
|
+
content_type: 'access-log',
|
|
134
|
+
};
|
|
135
|
+
}
|
|
136
|
+
// Standard log deduplication
|
|
137
|
+
const seen = new Map();
|
|
138
|
+
const order = [];
|
|
139
|
+
for (const line of lines) {
|
|
140
|
+
const normalized = normalizeLogLine(line);
|
|
141
|
+
if (seen.has(normalized)) {
|
|
142
|
+
const entry = seen.get(normalized);
|
|
143
|
+
entry.count++;
|
|
144
|
+
entry.lastLine = line;
|
|
145
|
+
}
|
|
146
|
+
else {
|
|
147
|
+
seen.set(normalized, { firstLine: line, count: 1, lastLine: line });
|
|
148
|
+
order.push(normalized);
|
|
149
|
+
}
|
|
150
|
+
}
|
|
151
|
+
const outputLines = [];
|
|
152
|
+
for (const key of order) {
|
|
153
|
+
const { firstLine, count } = seen.get(key);
|
|
154
|
+
if (count > 1) {
|
|
155
|
+
outputLines.push(`[×${count}] ${firstLine}`);
|
|
156
|
+
}
|
|
157
|
+
else {
|
|
158
|
+
outputLines.push(firstLine);
|
|
159
|
+
}
|
|
160
|
+
}
|
|
161
|
+
const summary = outputLines.join('\n');
|
|
162
|
+
const tokensSummarized = (0, utils_js_1.estimateTokens)(summary);
|
|
163
|
+
return {
|
|
164
|
+
summary,
|
|
165
|
+
tokens_original: tokensOriginal,
|
|
166
|
+
tokens_summarized: tokensSummarized,
|
|
167
|
+
savings_pct: tokensOriginal > 0 ? Math.round((1 - tokensSummarized / tokensOriginal) * 100) : 0,
|
|
168
|
+
content_type: 'log',
|
|
169
|
+
};
|
|
170
|
+
}
|
|
171
|
+
}
|
|
172
|
+
exports.LogSummarizer = LogSummarizer;
|
|
173
|
+
//# sourceMappingURL=log-summarizer.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"log-summarizer.js","sourceRoot":"","sources":["../../../src/plugins/summarizers/log-summarizer.ts"],"names":[],"mappings":";;;AACA,kDAAqD;AAErD,MAAM,sBAAsB,GAAG,EAAE,CAAC;AAElC,0CAA0C;AAC1C,MAAM,kBAAkB,GAAG;IACzB,mBAAmB,EAAI,uBAAuB;IAC9C,mBAAmB,EAAI,iBAAiB;CACzC,CAAC;AAEF,MAAM,kBAAkB,GAAG;IACzB,yCAAyC,EAAG,oBAAoB;IAChE,yCAAyC,EAAG,aAAa;CAC1D,CAAC;AAEF,6CAA6C;AAC7C,8FAA8F;AAC9F,MAAM,gBAAgB,GAAG,oIAAoI,CAAC;AAE9J,6DAA6D;AAC7D,SAAS,gBAAgB,CAAC,IAAY;IACpC,OAAO,IAAI;QACT,2FAA2F;SAC1F,OAAO,CAAC,yEAAyE,EAAE,aAAa,CAAC;QAClG,wBAAwB;SACvB,OAAO,CAAC,oBAAoB,EAAE,QAAQ,CAAC;QACxC,0BAA0B;SACzB,OAAO,CAAC,8BAA8B,EAAE,QAAQ,CAAC;QAClD,yDAAyD;SACxD,OAAO,CAAC,gBAAgB,EAAE,MAAM,CAAC;SACjC,IAAI,EAAE,CAAC;AACZ,CAAC;AAED,SAAS,WAAW,CAAC,KAAe;IAClC,MAAM,MAAM,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IACnE,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC;QAAE,OAAO,KAAK,CAAC;IACpC,IAAI,UAAU,GAAG,CAAC,CAAC;IACnB,KAAK,MAAM,IAAI,IAAI,MAAM,EAAE,CAAC;QAC1B,IAAI,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC;YAAE,UAAU,EAAE,CAAC;IAChD,CAAC;IACD,OAAO,UAAU,GAAG,MAAM,CAAC,MAAM,IAAI,GAAG,CAAC;AAC3C,CAAC;AAED,SAAS,kBAAkB,CAAC,OAAe;IACzC,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IACnE,MAAM,UAAU,GAAG,IAAI,GAAG,EAAkB,CAAC;IAC7C,MAAM,UAAU,GAAG,IAAI,GAAG,EAAkB,CAAC;IAC7C,MAAM,YAAY,GAAG,IAAI,GAAG,EAAkB,CAAC;IAC/C,IAAI,UAAU,GAAG,CAAC,CAAC;IAEnB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,CAAC,GAAG,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACtC,IAAI,CAAC,EAAE,CAAC;YACN,MAAM,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;YACpB,MAAM,QAAQ,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;YACtB,MAAM,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;YACpB,MAAM,KAAK,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YAEjC,UAAU,CAAC,GAAG,CAAC,MAAM,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;YAC1D,MAAM,SAAS,GAAG,GAAG,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC;YACnC,UAAU,CAAC,GAAG,CAAC,SAAS,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;YAChE,MAAM,GAAG,GAAG,GAAG,MAAM,IAAI,QAAQ,EAAE,CAAC;YACpC,YAAY,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,YAAY,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;YACxD,UAAU,IAAI,KAAK,CAAC;QACtB,CAAC;IACH,CAAC;IAED,MAAM,KAAK,GAAG;QACZ,yBAAyB,KAAK,CAAC,MAAM,YAAY;QACjD,EAAE;QACF,YAAY;QACZ,GAAG,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,OAAO,EAAE,CAAC;aAChC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;aAC3B,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC;QAClC,EAAE;QACF,iBAAiB;QACjB,GAAG,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,OAAO,EAAE,CAAC;aAChC,IAAI,EAAE;aACN,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC;QAClC,EAAE;QACF,qBAAqB,YAAY,CAAC,IAAI,UAAU;QAChD,GAAG,KAAK,CAAC,IAAI,CAAC,YAAY,CAAC,OAAO,EAAE,CAAC;aAClC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;aAC3B,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC;aACZ,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC;QAClC,EAAE;QACF,gBAAgB,CAAC,UAAU,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,WAAW;KAC1D,CAAC;IAEF,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC;AAED,MAAa,aAAa;IACxB,IAAI,GAAG,gBAAgB,CAAC;IACxB,OAAO,GAAG,OAAO,CAAC;IAClB,IAAI,GAAG,YAAqB,CAAC;IAC7B,YAAY,GAAG,CAAC,KAAK,EAAE,QAAQ,EAAE,iBAAiB,EAAE,YAAY,CAAC,CAAC;IAElE,KAAK,CAAC,IAAI,CAAC,OAAqB,IAAkB,CAAC;IACnD,KAAK,CAAC,OAAO,KAAmB,CAAC;IAEjC,MAAM,CAAC,OAAe;QACpB,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QACnE,IAAI,KAAK,CAAC,MAAM,GAAG,sBAAsB;YAAE,OAAO,KAAK,CAAC;QAExD,oCAAoC;QACpC,IAAI,WAAW,CAAC,KAAK,CAAC;YAAE,OAAO,IAAI,CAAC;QAEpC,4DAA4D;QAC5D,IAAI,UAAU,GAAG,CAAC,CAAC;QACnB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,MAAM,YAAY,GAAG,kBAAkB,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;YAChE,MAAM,WAAW,GAAG,kBAAkB,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;YAC/D,IAAI,YAAY,IAAI,WAAW;gBAAE,UAAU,EAAE,CAAC;QAChD,CAAC;QAED,mDAAmD;QACnD,OAAO,UAAU,GAAG,KAAK,CAAC,MAAM,IAAI,GAAG,CAAC;IAC1C,CAAC;IAED,KAAK,CAAC,SAAS,CAAC,OAAe,EAAE,KAAoB;QACnD,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAClC,MAAM,cAAc,GAAG,IAAA,yBAAc,EAAC,OAAO,CAAC,CAAC;QAE/C,IAAI,KAAK,CAAC,MAAM,GAAG,sBAAsB,EAAE,CAAC;YAC1C,OAAO;gBACL,OAAO,EAAE,OAAO;gBAChB,eAAe,EAAE,cAAc;gBAC/B,iBAAiB,EAAE,cAAc;gBACjC,WAAW,EAAE,CAAC;gBACd,YAAY,EAAE,KAAK;aACpB,CAAC;QACJ,CAAC;QAED,gDAAgD;QAChD,MAAM,QAAQ,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QACxD,IAAI,WAAW,CAAC,QAAQ,CAAC,EAAE,CAAC;YAC1B,MAAM,OAAO,GAAG,kBAAkB,CAAC,OAAO,CAAC,CAAC;YAC5C,MAAM,gBAAgB,GAAG,IAAA,yBAAc,EAAC,OAAO,CAAC,CAAC;YACjD,OAAO;gBACL,OAAO;gBACP,eAAe,EAAE,cAAc;gBAC/B,iBAAiB,EAAE,gBAAgB;gBACnC,WAAW,EAAE,cAAc,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,gBAAgB,GAAG,cAAc,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;gBAC/F,YAAY,EAAE,YAAY;aAC3B,CAAC;QACJ,CAAC;QAED,6BAA6B;QAC7B,MAAM,IAAI,GAAG,IAAI,GAAG,EAAkE,CAAC;QACvF,MAAM,KAAK,GAAa,EAAE,CAAC;QAE3B,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,MAAM,UAAU,GAAG,gBAAgB,CAAC,IAAI,CAAC,CAAC;YAC1C,IAAI,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,EAAE,CAAC;gBACzB,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,UAAU,CAAE,CAAC;gBACpC,KAAK,CAAC,KAAK,EAAE,CAAC;gBACd,KAAK,CAAC,QAAQ,GAAG,IAAI,CAAC;YACxB,CAAC;iBAAM,CAAC;gBACN,IAAI,CAAC,GAAG,CAAC,UAAU,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC;gBACpE,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;YACzB,CAAC;QACH,CAAC;QAED,MAAM,WAAW,GAAa,EAAE,CAAC;QACjC,KAAK,MAAM,GAAG,IAAI,KAAK,EAAE,CAAC;YACxB,MAAM,EAAE,SAAS,EAAE,KAAK,EAAE,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,CAAE,CAAC;YAC5C,IAAI,KAAK,GAAG,CAAC,EAAE,CAAC;gBACd,WAAW,CAAC,IAAI,CAAC,KAAK,KAAK,KAAK,SAAS,EAAE,CAAC,CAAC;YAC/C,CAAC;iBAAM,CAAC;gBACN,WAAW,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YAC9B,CAAC;QACH,CAAC;QAED,MAAM,OAAO,GAAG,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACvC,MAAM,gBAAgB,GAAG,IAAA,yBAAc,EAAC,OAAO,CAAC,CAAC;QAEjD,OAAO;YACL,OAAO;YACP,eAAe,EAAE,cAAc;YAC/B,iBAAiB,EAAE,gBAAgB;YACnC,WAAW,EAAE,cAAc,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,gBAAgB,GAAG,cAAc,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;YAC/F,YAAY,EAAE,KAAK;SACpB,CAAC;IACJ,CAAC;CACF;AA7FD,sCA6FC"}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import type { SummarizerPlugin, PluginConfig, SummaryResult, SummarizeOpts } from '../../core/types.js';
|
|
2
|
+
export declare class MarkdownSummarizer implements SummarizerPlugin {
|
|
3
|
+
name: string;
|
|
4
|
+
version: string;
|
|
5
|
+
type: "summarizer";
|
|
6
|
+
contentTypes: string[];
|
|
7
|
+
init(_config: PluginConfig): Promise<void>;
|
|
8
|
+
destroy(): Promise<void>;
|
|
9
|
+
detect(content: string): boolean;
|
|
10
|
+
summarize(content: string, _opts: SummarizeOpts): Promise<SummaryResult>;
|
|
11
|
+
}
|
|
12
|
+
//# sourceMappingURL=markdown-summarizer.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"markdown-summarizer.d.ts","sourceRoot":"","sources":["../../../src/plugins/summarizers/markdown-summarizer.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,gBAAgB,EAAE,YAAY,EAAE,aAAa,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AAKxG,qBAAa,kBAAmB,YAAW,gBAAgB;IACzD,IAAI,SAAyB;IAC7B,OAAO,SAAW;IAClB,IAAI,EAAG,YAAY,CAAU;IAC7B,YAAY,WAAgB;IAEtB,IAAI,CAAC,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC;IAC1C,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;IAE9B,MAAM,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO;IAU1B,SAAS,CAAC,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,aAAa,GAAG,OAAO,CAAC,aAAa,CAAC;CAuD/E"}
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.MarkdownSummarizer = void 0;
|
|
4
|
+
const utils_js_1 = require("../../core/utils.js");
|
|
5
|
+
const HEADING_REGEX = /^#{1,6}\s/;
|
|
6
|
+
class MarkdownSummarizer {
|
|
7
|
+
name = 'markdown-summarizer';
|
|
8
|
+
version = '1.0.0';
|
|
9
|
+
type = 'summarizer';
|
|
10
|
+
contentTypes = ['markdown'];
|
|
11
|
+
async init(_config) { }
|
|
12
|
+
async destroy() { }
|
|
13
|
+
detect(content) {
|
|
14
|
+
const lines = content.split('\n');
|
|
15
|
+
let headingCount = 0;
|
|
16
|
+
for (const line of lines) {
|
|
17
|
+
if (HEADING_REGEX.test(line))
|
|
18
|
+
headingCount++;
|
|
19
|
+
if (headingCount >= 2)
|
|
20
|
+
return true;
|
|
21
|
+
}
|
|
22
|
+
return false;
|
|
23
|
+
}
|
|
24
|
+
async summarize(content, _opts) {
|
|
25
|
+
const lines = content.split('\n');
|
|
26
|
+
const tokensOriginal = (0, utils_js_1.estimateTokens)(content);
|
|
27
|
+
// Extract title (first heading)
|
|
28
|
+
let title = '';
|
|
29
|
+
for (const line of lines) {
|
|
30
|
+
if (HEADING_REGEX.test(line)) {
|
|
31
|
+
title = line.replace(/^#+\s/, '').trim();
|
|
32
|
+
break;
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
// Build heading tree
|
|
36
|
+
const headings = [];
|
|
37
|
+
for (const line of lines) {
|
|
38
|
+
if (HEADING_REGEX.test(line)) {
|
|
39
|
+
headings.push(line.trim());
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
// Count code blocks
|
|
43
|
+
let codeBlockCount = 0;
|
|
44
|
+
for (const line of lines) {
|
|
45
|
+
if (line.trim().startsWith('```'))
|
|
46
|
+
codeBlockCount++;
|
|
47
|
+
}
|
|
48
|
+
codeBlockCount = Math.floor(codeBlockCount / 2);
|
|
49
|
+
// Count links
|
|
50
|
+
const linkMatches = content.match(/\[([^\]]*)\]\([^)]*\)/g);
|
|
51
|
+
const linkCount = linkMatches ? linkMatches.length : 0;
|
|
52
|
+
const summaryParts = [
|
|
53
|
+
`# Document: ${title || '(untitled)'}`,
|
|
54
|
+
'',
|
|
55
|
+
`## Structure (${headings.length} headings)`,
|
|
56
|
+
...headings.map(h => ` ${h}`),
|
|
57
|
+
'',
|
|
58
|
+
`## Stats`,
|
|
59
|
+
`- Lines: ${lines.length}`,
|
|
60
|
+
`- Code blocks: ${codeBlockCount}`,
|
|
61
|
+
`- Links: ${linkCount}`,
|
|
62
|
+
];
|
|
63
|
+
const summary = summaryParts.join('\n');
|
|
64
|
+
const tokensSummarized = (0, utils_js_1.estimateTokens)(summary);
|
|
65
|
+
return {
|
|
66
|
+
summary,
|
|
67
|
+
tokens_original: tokensOriginal,
|
|
68
|
+
tokens_summarized: tokensSummarized,
|
|
69
|
+
savings_pct: tokensOriginal > 0 ? Math.round((1 - tokensSummarized / tokensOriginal) * 100) : 0,
|
|
70
|
+
content_type: 'markdown',
|
|
71
|
+
};
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
exports.MarkdownSummarizer = MarkdownSummarizer;
|
|
75
|
+
//# sourceMappingURL=markdown-summarizer.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"markdown-summarizer.js","sourceRoot":"","sources":["../../../src/plugins/summarizers/markdown-summarizer.ts"],"names":[],"mappings":";;;AACA,kDAAqD;AAErD,MAAM,aAAa,GAAG,WAAW,CAAC;AAElC,MAAa,kBAAkB;IAC7B,IAAI,GAAG,qBAAqB,CAAC;IAC7B,OAAO,GAAG,OAAO,CAAC;IAClB,IAAI,GAAG,YAAqB,CAAC;IAC7B,YAAY,GAAG,CAAC,UAAU,CAAC,CAAC;IAE5B,KAAK,CAAC,IAAI,CAAC,OAAqB,IAAkB,CAAC;IACnD,KAAK,CAAC,OAAO,KAAmB,CAAC;IAEjC,MAAM,CAAC,OAAe;QACpB,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAClC,IAAI,YAAY,GAAG,CAAC,CAAC;QACrB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,IAAI,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC;gBAAE,YAAY,EAAE,CAAC;YAC7C,IAAI,YAAY,IAAI,CAAC;gBAAE,OAAO,IAAI,CAAC;QACrC,CAAC;QACD,OAAO,KAAK,CAAC;IACf,CAAC;IAED,KAAK,CAAC,SAAS,CAAC,OAAe,EAAE,KAAoB;QACnD,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAClC,MAAM,cAAc,GAAG,IAAA,yBAAc,EAAC,OAAO,CAAC,CAAC;QAE/C,gCAAgC;QAChC,IAAI,KAAK,GAAG,EAAE,CAAC;QACf,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,IAAI,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;gBAC7B,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;gBACzC,MAAM;YACR,CAAC;QACH,CAAC;QAED,qBAAqB;QACrB,MAAM,QAAQ,GAAa,EAAE,CAAC;QAC9B,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,IAAI,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;gBAC7B,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;YAC7B,CAAC;QACH,CAAC;QAED,oBAAoB;QACpB,IAAI,cAAc,GAAG,CAAC,CAAC;QACvB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC,UAAU,CAAC,KAAK,CAAC;gBAAE,cAAc,EAAE,CAAC;QACtD,CAAC;QACD,cAAc,GAAG,IAAI,CAAC,KAAK,CAAC,cAAc,GAAG,CAAC,CAAC,CAAC;QAEhD,cAAc;QACd,MAAM,WAAW,GAAG,OAAO,CAAC,KAAK,CAAC,wBAAwB,CAAC,CAAC;QAC5D,MAAM,SAAS,GAAG,WAAW,CAAC,CAAC,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;QAEvD,MAAM,YAAY,GAAG;YACnB,eAAe,KAAK,IAAI,YAAY,EAAE;YACtC,EAAE;YACF,iBAAiB,QAAQ,CAAC,MAAM,YAAY;YAC5C,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,KAAK,CAAC,EAAE,CAAC;YAC9B,EAAE;YACF,UAAU;YACV,YAAY,KAAK,CAAC,MAAM,EAAE;YAC1B,kBAAkB,cAAc,EAAE;YAClC,YAAY,SAAS,EAAE;SACxB,CAAC;QAEF,MAAM,OAAO,GAAG,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACxC,MAAM,gBAAgB,GAAG,IAAA,yBAAc,EAAC,OAAO,CAAC,CAAC;QAEjD,OAAO;YACL,OAAO;YACP,eAAe,EAAE,cAAc;YAC/B,iBAAiB,EAAE,gBAAgB;YACnC,WAAW,EAAE,cAAc,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,gBAAgB,GAAG,cAAc,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;YAC/F,YAAY,EAAE,UAAU;SACzB,CAAC;IACJ,CAAC;CACF;AA1ED,gDA0EC"}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import type { SummarizerPlugin, PluginConfig, SummaryResult, SummarizeOpts } from '../../core/types.js';
|
|
2
|
+
export declare class NetworkSummarizer implements SummarizerPlugin {
|
|
3
|
+
name: string;
|
|
4
|
+
version: string;
|
|
5
|
+
type: "summarizer";
|
|
6
|
+
contentTypes: string[];
|
|
7
|
+
init(_config: PluginConfig): Promise<void>;
|
|
8
|
+
destroy(): Promise<void>;
|
|
9
|
+
detect(content: string): boolean;
|
|
10
|
+
summarize(content: string, _opts: SummarizeOpts): Promise<SummaryResult>;
|
|
11
|
+
}
|
|
12
|
+
//# sourceMappingURL=network-summarizer.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"network-summarizer.d.ts","sourceRoot":"","sources":["../../../src/plugins/summarizers/network-summarizer.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,gBAAgB,EAAE,YAAY,EAAE,aAAa,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AAOxG,qBAAa,iBAAkB,YAAW,gBAAgB;IACxD,IAAI,SAAwB;IAC5B,OAAO,SAAW;IAClB,IAAI,EAAG,YAAY,CAAU;IAC7B,YAAY,WAAe;IAErB,IAAI,CAAC,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC;IAC1C,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;IAE9B,MAAM,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO;IAU1B,SAAS,CAAC,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,aAAa,GAAG,OAAO,CAAC,aAAa,CAAC;CAsD/E"}
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.NetworkSummarizer = void 0;
|
|
4
|
+
const utils_js_1 = require("../../core/utils.js");
|
|
5
|
+
const HTTP_METHOD_REGEX = /\b(GET|POST|PUT|DELETE|PATCH)\b/g;
|
|
6
|
+
const STATUS_CODE_REGEX = /\b([2-5]\d{2})\b/g;
|
|
7
|
+
const ENDPOINT_REGEX = /(?:GET|POST|PUT|DELETE|PATCH)\s+(\/\S+)/g;
|
|
8
|
+
class NetworkSummarizer {
|
|
9
|
+
name = 'network-summarizer';
|
|
10
|
+
version = '1.0.0';
|
|
11
|
+
type = 'summarizer';
|
|
12
|
+
contentTypes = ['network'];
|
|
13
|
+
async init(_config) { }
|
|
14
|
+
async destroy() { }
|
|
15
|
+
detect(content) {
|
|
16
|
+
// Skip small content — not worth summarizing
|
|
17
|
+
const lines = content.split('\n').filter(l => l.trim().length > 0);
|
|
18
|
+
if (lines.length < 10)
|
|
19
|
+
return false;
|
|
20
|
+
const hasMethod = /\b(GET|POST|PUT|DELETE|PATCH)\b/.test(content);
|
|
21
|
+
const hasStatus = /\b[2-5]\d{2}\b/.test(content);
|
|
22
|
+
return hasMethod && hasStatus;
|
|
23
|
+
}
|
|
24
|
+
async summarize(content, _opts) {
|
|
25
|
+
const tokensOriginal = (0, utils_js_1.estimateTokens)(content);
|
|
26
|
+
// Method distribution
|
|
27
|
+
const methodDist = new Map();
|
|
28
|
+
let methodMatch;
|
|
29
|
+
const methodRegex = new RegExp(HTTP_METHOD_REGEX.source, 'g');
|
|
30
|
+
while ((methodMatch = methodRegex.exec(content)) !== null) {
|
|
31
|
+
const method = methodMatch[1];
|
|
32
|
+
methodDist.set(method, (methodDist.get(method) || 0) + 1);
|
|
33
|
+
}
|
|
34
|
+
// Status code distribution
|
|
35
|
+
const statusDist = new Map();
|
|
36
|
+
let statusMatch;
|
|
37
|
+
const statusRegex = new RegExp(STATUS_CODE_REGEX.source, 'g');
|
|
38
|
+
while ((statusMatch = statusRegex.exec(content)) !== null) {
|
|
39
|
+
const code = statusMatch[1];
|
|
40
|
+
const category = `${code[0]}xx`;
|
|
41
|
+
statusDist.set(category, (statusDist.get(category) || 0) + 1);
|
|
42
|
+
}
|
|
43
|
+
// Unique endpoints
|
|
44
|
+
const endpoints = new Set();
|
|
45
|
+
let endpointMatch;
|
|
46
|
+
const endpointRegex = new RegExp(ENDPOINT_REGEX.source, 'g');
|
|
47
|
+
while ((endpointMatch = endpointRegex.exec(content)) !== null) {
|
|
48
|
+
endpoints.add(endpointMatch[1]);
|
|
49
|
+
}
|
|
50
|
+
const summaryParts = [
|
|
51
|
+
`# Network Summary`,
|
|
52
|
+
'',
|
|
53
|
+
`## Methods`,
|
|
54
|
+
...Array.from(methodDist.entries()).map(([m, c]) => ` ${m}: ${c}`),
|
|
55
|
+
'',
|
|
56
|
+
`## Status codes`,
|
|
57
|
+
...Array.from(statusDist.entries()).sort().map(([s, c]) => ` ${s}: ${c}`),
|
|
58
|
+
'',
|
|
59
|
+
`## Unique endpoints: ${endpoints.size}`,
|
|
60
|
+
...Array.from(endpoints).slice(0, 20).map(e => ` ${e}`),
|
|
61
|
+
];
|
|
62
|
+
const summary = summaryParts.join('\n');
|
|
63
|
+
const tokensSummarized = (0, utils_js_1.estimateTokens)(summary);
|
|
64
|
+
return {
|
|
65
|
+
summary,
|
|
66
|
+
tokens_original: tokensOriginal,
|
|
67
|
+
tokens_summarized: tokensSummarized,
|
|
68
|
+
savings_pct: tokensOriginal > 0 ? Math.round((1 - tokensSummarized / tokensOriginal) * 100) : 0,
|
|
69
|
+
content_type: 'network',
|
|
70
|
+
};
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
exports.NetworkSummarizer = NetworkSummarizer;
|
|
74
|
+
//# sourceMappingURL=network-summarizer.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"network-summarizer.js","sourceRoot":"","sources":["../../../src/plugins/summarizers/network-summarizer.ts"],"names":[],"mappings":";;;AACA,kDAAqD;AAErD,MAAM,iBAAiB,GAAG,kCAAkC,CAAC;AAC7D,MAAM,iBAAiB,GAAG,mBAAmB,CAAC;AAC9C,MAAM,cAAc,GAAG,0CAA0C,CAAC;AAElE,MAAa,iBAAiB;IAC5B,IAAI,GAAG,oBAAoB,CAAC;IAC5B,OAAO,GAAG,OAAO,CAAC;IAClB,IAAI,GAAG,YAAqB,CAAC;IAC7B,YAAY,GAAG,CAAC,SAAS,CAAC,CAAC;IAE3B,KAAK,CAAC,IAAI,CAAC,OAAqB,IAAkB,CAAC;IACnD,KAAK,CAAC,OAAO,KAAmB,CAAC;IAEjC,MAAM,CAAC,OAAe;QACpB,6CAA6C;QAC7C,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QACnE,IAAI,KAAK,CAAC,MAAM,GAAG,EAAE;YAAE,OAAO,KAAK,CAAC;QAEpC,MAAM,SAAS,GAAG,iCAAiC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QAClE,MAAM,SAAS,GAAG,gBAAgB,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QACjD,OAAO,SAAS,IAAI,SAAS,CAAC;IAChC,CAAC;IAED,KAAK,CAAC,SAAS,CAAC,OAAe,EAAE,KAAoB;QACnD,MAAM,cAAc,GAAG,IAAA,yBAAc,EAAC,OAAO,CAAC,CAAC;QAE/C,sBAAsB;QACtB,MAAM,UAAU,GAAG,IAAI,GAAG,EAAkB,CAAC;QAC7C,IAAI,WAAW,CAAC;QAChB,MAAM,WAAW,GAAG,IAAI,MAAM,CAAC,iBAAiB,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;QAC9D,OAAO,CAAC,WAAW,GAAG,WAAW,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;YAC1D,MAAM,MAAM,GAAG,WAAW,CAAC,CAAC,CAAC,CAAC;YAC9B,UAAU,CAAC,GAAG,CAAC,MAAM,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QAC5D,CAAC;QAED,2BAA2B;QAC3B,MAAM,UAAU,GAAG,IAAI,GAAG,EAAkB,CAAC;QAC7C,IAAI,WAAW,CAAC;QAChB,MAAM,WAAW,GAAG,IAAI,MAAM,CAAC,iBAAiB,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;QAC9D,OAAO,CAAC,WAAW,GAAG,WAAW,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;YAC1D,MAAM,IAAI,GAAG,WAAW,CAAC,CAAC,CAAC,CAAC;YAC5B,MAAM,QAAQ,GAAG,GAAG,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC;YAChC,UAAU,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QAChE,CAAC;QAED,mBAAmB;QACnB,MAAM,SAAS,GAAG,IAAI,GAAG,EAAU,CAAC;QACpC,IAAI,aAAa,CAAC;QAClB,MAAM,aAAa,GAAG,IAAI,MAAM,CAAC,cAAc,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;QAC7D,OAAO,CAAC,aAAa,GAAG,aAAa,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;YAC9D,SAAS,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,CAAC;QAClC,CAAC;QAED,MAAM,YAAY,GAAG;YACnB,mBAAmB;YACnB,EAAE;YACF,YAAY;YACZ,GAAG,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,OAAO,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC;YACnE,EAAE;YACF,iBAAiB;YACjB,GAAG,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,OAAO,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC;YAC1E,EAAE;YACF,wBAAwB,SAAS,CAAC,IAAI,EAAE;YACxC,GAAG,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,KAAK,CAAC,EAAE,CAAC;SACzD,CAAC;QAEF,MAAM,OAAO,GAAG,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACxC,MAAM,gBAAgB,GAAG,IAAA,yBAAc,EAAC,OAAO,CAAC,CAAC;QAEjD,OAAO;YACL,OAAO;YACP,eAAe,EAAE,cAAc;YAC/B,iBAAiB,EAAE,gBAAgB;YACnC,WAAW,EAAE,cAAc,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,gBAAgB,GAAG,cAAc,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;YAC/F,YAAY,EAAE,SAAS;SACxB,CAAC;IACJ,CAAC;CACF;AAzED,8CAyEC"}
|