@sitrozyi/repomix-semantic-compressor 0.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +105 -0
- package/bin/cli.mjs +16 -0
- package/bin/mcp-server.mjs +8 -0
- package/package.json +65 -0
- package/src/ast.mjs +489 -0
- package/src/core.mjs +502 -0
- package/src/extractor.mjs +89 -0
- package/src/imports.mjs +228 -0
- package/src/mcp.mjs +223 -0
- package/src/optimizers.mjs +870 -0
- package/src/worker.mjs +28 -0
|
@@ -0,0 +1,870 @@
|
|
|
1
|
+
import postcss from 'postcss';
|
|
2
|
+
|
|
3
|
+
export function optimizeHTML(html) {
|
|
4
|
+
return html
|
|
5
|
+
.replace(/<svg[\s\S]*?<\/svg>/gi, (svg) => {
|
|
6
|
+
if (svg.length > 150) {
|
|
7
|
+
const matchId = svg.match(/id="([^"]+)"/);
|
|
8
|
+
const idAttr = matchId ? ` id="${matchId[1]}"` : '';
|
|
9
|
+
return `<svg${idAttr}><!-- [SVG Icon Path Omitted] --></svg>`;
|
|
10
|
+
}
|
|
11
|
+
return svg;
|
|
12
|
+
})
|
|
13
|
+
.replace(/data:(image|font)\/[^;]+;base64,[a-zA-Z0-9+/=]+/g, 'data:$1/...[base64 omitted]...')
|
|
14
|
+
.replace(/[ \t]{2,}/g, ' ');
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
const LAYOUT_PROPS = new Set([
|
|
18
|
+
'display',
|
|
19
|
+
'position',
|
|
20
|
+
'top',
|
|
21
|
+
'bottom',
|
|
22
|
+
'left',
|
|
23
|
+
'right',
|
|
24
|
+
'grid-template-columns',
|
|
25
|
+
'grid-template-rows',
|
|
26
|
+
'grid-template-areas',
|
|
27
|
+
'flex-direction',
|
|
28
|
+
'flex-wrap',
|
|
29
|
+
'align-items',
|
|
30
|
+
'justify-content',
|
|
31
|
+
'gap',
|
|
32
|
+
'z-index',
|
|
33
|
+
'overflow',
|
|
34
|
+
'visibility'
|
|
35
|
+
]);
|
|
36
|
+
|
|
37
|
+
export function summarizeCSS(cssCode) {
|
|
38
|
+
if (!cssCode || !cssCode.trim()) return '/* Empty stylesheet */';
|
|
39
|
+
|
|
40
|
+
let root;
|
|
41
|
+
try {
|
|
42
|
+
root = postcss.parse(cssCode, { from: undefined });
|
|
43
|
+
} catch {
|
|
44
|
+
return '/* Invalid CSS stylesheet */';
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
const rootVariables = [];
|
|
48
|
+
const layoutRules = [];
|
|
49
|
+
const decorativeClasses = new Set();
|
|
50
|
+
|
|
51
|
+
function extractRule(ruleNode) {
|
|
52
|
+
const keptDecls = [];
|
|
53
|
+
const nestedRules = [];
|
|
54
|
+
|
|
55
|
+
for (const child of ruleNode.nodes || []) {
|
|
56
|
+
if (child.type === 'decl') {
|
|
57
|
+
const prop = child.prop.toLowerCase();
|
|
58
|
+
if (LAYOUT_PROPS.has(prop) || prop.startsWith('--')) {
|
|
59
|
+
keptDecls.push(`${child.prop}: ${child.value}`);
|
|
60
|
+
}
|
|
61
|
+
} else if (child.type === 'rule') {
|
|
62
|
+
const sub = extractRule(child);
|
|
63
|
+
if (sub) nestedRules.push(sub);
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
if (keptDecls.length > 0 || nestedRules.length > 0) {
|
|
68
|
+
const parts = [];
|
|
69
|
+
if (keptDecls.length > 0) {
|
|
70
|
+
parts.push(keptDecls.join('; '));
|
|
71
|
+
}
|
|
72
|
+
if (nestedRules.length > 0) {
|
|
73
|
+
parts.push(nestedRules.join('\n '));
|
|
74
|
+
}
|
|
75
|
+
return `${ruleNode.selector} { ${parts.join('; ')} }`;
|
|
76
|
+
}
|
|
77
|
+
return null;
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
function processContainer(container) {
|
|
81
|
+
if (!container || !container.nodes) return;
|
|
82
|
+
|
|
83
|
+
for (const node of container.nodes) {
|
|
84
|
+
if (node.type === 'comment') continue;
|
|
85
|
+
|
|
86
|
+
if (node.type === 'atrule') {
|
|
87
|
+
const atName = node.name.toLowerCase();
|
|
88
|
+
if (['media', 'supports', 'container'].includes(atName)) {
|
|
89
|
+
const subLayoutRules = [];
|
|
90
|
+
for (const subNode of node.nodes || []) {
|
|
91
|
+
if (subNode.type === 'rule') {
|
|
92
|
+
const ruleResult = extractRule(subNode);
|
|
93
|
+
if (ruleResult) subLayoutRules.push(ruleResult);
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
if (subLayoutRules.length > 0) {
|
|
97
|
+
layoutRules.push(`@${node.name} ${node.params} {\n ${subLayoutRules.join('\n ')}\n}`);
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
continue;
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
if (node.type === 'rule') {
|
|
104
|
+
const selector = node.selector ? node.selector.trim() : '';
|
|
105
|
+
if (!selector) continue;
|
|
106
|
+
|
|
107
|
+
const isRootScope = selector === ':root' || selector === ':host' || selector.startsWith(':root');
|
|
108
|
+
if (isRootScope) {
|
|
109
|
+
const varDecls = [];
|
|
110
|
+
node.walkDecls((decl) => {
|
|
111
|
+
if (decl.prop.startsWith('--')) {
|
|
112
|
+
varDecls.push(`${decl.prop}: ${decl.value}`);
|
|
113
|
+
}
|
|
114
|
+
});
|
|
115
|
+
if (varDecls.length > 0) {
|
|
116
|
+
rootVariables.push(`${selector} {\n ${varDecls.join(';\n ')};\n}`);
|
|
117
|
+
}
|
|
118
|
+
continue;
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
const ruleResult = extractRule(node);
|
|
122
|
+
if (ruleResult) {
|
|
123
|
+
layoutRules.push(ruleResult);
|
|
124
|
+
} else {
|
|
125
|
+
const classMatches = selector.match(/\.[a-zA-Z0-9_-]+/g);
|
|
126
|
+
if (classMatches) {
|
|
127
|
+
for (const cls of classMatches) decorativeClasses.add(cls);
|
|
128
|
+
}
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
processContainer(root);
|
|
135
|
+
|
|
136
|
+
const sections = [];
|
|
137
|
+
if (rootVariables.length > 0) {
|
|
138
|
+
sections.push(`/* Design Tokens & CSS Variables */\n${rootVariables.join('\n\n')}`);
|
|
139
|
+
}
|
|
140
|
+
if (layoutRules.length > 0) {
|
|
141
|
+
sections.push(`/* Layout & Structural Rules */\n${layoutRules.join('\n')}`);
|
|
142
|
+
}
|
|
143
|
+
if (decorativeClasses.size > 0) {
|
|
144
|
+
sections.push(`/* Decorative/Component Classes (${decorativeClasses.size} classes) */\n` + Array.from(decorativeClasses).join(', '));
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
return sections.length > 0 ? sections.join('\n\n') : '/* No distinct layout or token definitions found */';
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
export function processJSON(jsonStr) {
|
|
151
|
+
try {
|
|
152
|
+
const cleaned = jsonStr.replace(/,(\s*[}\]])/g, '$1');
|
|
153
|
+
return JSON.stringify(JSON.parse(cleaned));
|
|
154
|
+
} catch {
|
|
155
|
+
return jsonStr;
|
|
156
|
+
}
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
export function splitSQLStatements(sqlCode) {
|
|
160
|
+
const statements = [];
|
|
161
|
+
let current = '';
|
|
162
|
+
let inSingleQuote = false;
|
|
163
|
+
let inDoubleQuote = false;
|
|
164
|
+
let inBacktick = false;
|
|
165
|
+
let inLineComment = false;
|
|
166
|
+
let inBlockComment = false;
|
|
167
|
+
let dollarTag = null;
|
|
168
|
+
|
|
169
|
+
for (let i = 0; i < sqlCode.length; i++) {
|
|
170
|
+
const char = sqlCode[i];
|
|
171
|
+
const nextChar = sqlCode[i + 1];
|
|
172
|
+
|
|
173
|
+
if (inLineComment) {
|
|
174
|
+
current += char;
|
|
175
|
+
if (char === '\n') {
|
|
176
|
+
inLineComment = false;
|
|
177
|
+
}
|
|
178
|
+
continue;
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
if (inBlockComment) {
|
|
182
|
+
current += char;
|
|
183
|
+
if (char === '*' && nextChar === '/') {
|
|
184
|
+
current += nextChar;
|
|
185
|
+
i++;
|
|
186
|
+
inBlockComment = false;
|
|
187
|
+
}
|
|
188
|
+
continue;
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
if (dollarTag !== null) {
|
|
192
|
+
if (char === '$' && sqlCode.startsWith(dollarTag, i)) {
|
|
193
|
+
current += dollarTag;
|
|
194
|
+
i += dollarTag.length - 1;
|
|
195
|
+
dollarTag = null;
|
|
196
|
+
} else {
|
|
197
|
+
current += char;
|
|
198
|
+
}
|
|
199
|
+
continue;
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
if (inSingleQuote) {
|
|
203
|
+
current += char;
|
|
204
|
+
if (char === '\\') {
|
|
205
|
+
if (nextChar) {
|
|
206
|
+
current += nextChar;
|
|
207
|
+
i++;
|
|
208
|
+
}
|
|
209
|
+
} else if (char === "'") {
|
|
210
|
+
if (nextChar === "'") {
|
|
211
|
+
current += nextChar;
|
|
212
|
+
i++;
|
|
213
|
+
} else {
|
|
214
|
+
inSingleQuote = false;
|
|
215
|
+
}
|
|
216
|
+
}
|
|
217
|
+
continue;
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
if (inDoubleQuote) {
|
|
221
|
+
current += char;
|
|
222
|
+
if (char === '\\') {
|
|
223
|
+
if (nextChar) {
|
|
224
|
+
current += nextChar;
|
|
225
|
+
i++;
|
|
226
|
+
}
|
|
227
|
+
} else if (char === '"') {
|
|
228
|
+
if (nextChar === '"') {
|
|
229
|
+
current += nextChar;
|
|
230
|
+
i++;
|
|
231
|
+
} else {
|
|
232
|
+
inDoubleQuote = false;
|
|
233
|
+
}
|
|
234
|
+
}
|
|
235
|
+
continue;
|
|
236
|
+
}
|
|
237
|
+
|
|
238
|
+
if (inBacktick) {
|
|
239
|
+
current += char;
|
|
240
|
+
if (char === '`') {
|
|
241
|
+
inBacktick = false;
|
|
242
|
+
}
|
|
243
|
+
continue;
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
if (char === '-' && nextChar === '-') {
|
|
247
|
+
inLineComment = true;
|
|
248
|
+
current += char + nextChar;
|
|
249
|
+
i++;
|
|
250
|
+
continue;
|
|
251
|
+
}
|
|
252
|
+
|
|
253
|
+
if (char === '/' && nextChar === '*') {
|
|
254
|
+
inBlockComment = true;
|
|
255
|
+
current += char + nextChar;
|
|
256
|
+
i++;
|
|
257
|
+
continue;
|
|
258
|
+
}
|
|
259
|
+
|
|
260
|
+
if (char === '$') {
|
|
261
|
+
let tagEnd = -1;
|
|
262
|
+
for (let j = i + 1; j < sqlCode.length && j <= i + 64; j++) {
|
|
263
|
+
const c = sqlCode[j];
|
|
264
|
+
if (c === '$') {
|
|
265
|
+
tagEnd = j;
|
|
266
|
+
break;
|
|
267
|
+
}
|
|
268
|
+
if (!(/[a-zA-Z0-9_]/).test(c)) {
|
|
269
|
+
break;
|
|
270
|
+
}
|
|
271
|
+
}
|
|
272
|
+
if (tagEnd !== -1) {
|
|
273
|
+
dollarTag = sqlCode.substring(i, tagEnd + 1);
|
|
274
|
+
current += dollarTag;
|
|
275
|
+
i = tagEnd;
|
|
276
|
+
continue;
|
|
277
|
+
}
|
|
278
|
+
}
|
|
279
|
+
|
|
280
|
+
if (char === "'") {
|
|
281
|
+
inSingleQuote = true;
|
|
282
|
+
current += char;
|
|
283
|
+
continue;
|
|
284
|
+
}
|
|
285
|
+
|
|
286
|
+
if (char === '"') {
|
|
287
|
+
inDoubleQuote = true;
|
|
288
|
+
current += char;
|
|
289
|
+
continue;
|
|
290
|
+
}
|
|
291
|
+
|
|
292
|
+
if (char === '`') {
|
|
293
|
+
inBacktick = true;
|
|
294
|
+
current += char;
|
|
295
|
+
continue;
|
|
296
|
+
}
|
|
297
|
+
|
|
298
|
+
if (char === ';') {
|
|
299
|
+
const trimmed = current.trim();
|
|
300
|
+
if (trimmed) {
|
|
301
|
+
statements.push(trimmed);
|
|
302
|
+
}
|
|
303
|
+
current = '';
|
|
304
|
+
continue;
|
|
305
|
+
}
|
|
306
|
+
|
|
307
|
+
current += char;
|
|
308
|
+
}
|
|
309
|
+
|
|
310
|
+
const trimmed = current.trim();
|
|
311
|
+
if (trimmed) {
|
|
312
|
+
statements.push(trimmed);
|
|
313
|
+
}
|
|
314
|
+
|
|
315
|
+
return statements;
|
|
316
|
+
}
|
|
317
|
+
|
|
318
|
+
export function optimizeSQL(sqlCode) {
|
|
319
|
+
const statements = splitSQLStatements(sqlCode);
|
|
320
|
+
|
|
321
|
+
const keptStatements = [];
|
|
322
|
+
const insertCountsByTable = new Map();
|
|
323
|
+
|
|
324
|
+
for (const stmt of statements) {
|
|
325
|
+
const upper = stmt.toUpperCase();
|
|
326
|
+
|
|
327
|
+
const insertMatch = stmt.match(/^INSERT\s+INTO\s+([`"'\w.]+)/i);
|
|
328
|
+
|
|
329
|
+
if (insertMatch) {
|
|
330
|
+
const tableName = insertMatch[1];
|
|
331
|
+
const count = (insertCountsByTable.get(tableName) || 0) + 1;
|
|
332
|
+
insertCountsByTable.set(tableName, count);
|
|
333
|
+
|
|
334
|
+
if (count <= 2) {
|
|
335
|
+
keptStatements.push(`${stmt};`);
|
|
336
|
+
}
|
|
337
|
+
} else {
|
|
338
|
+
keptStatements.push(`${stmt};`);
|
|
339
|
+
}
|
|
340
|
+
}
|
|
341
|
+
|
|
342
|
+
const omittedSummaries = [];
|
|
343
|
+
for (const [table, total] of insertCountsByTable.entries()) {
|
|
344
|
+
if (total > 2) {
|
|
345
|
+
omittedSummaries.push(`/* ... ${total - 2} redundant INSERT statements omitted for ${table} ... */`);
|
|
346
|
+
}
|
|
347
|
+
}
|
|
348
|
+
|
|
349
|
+
if (omittedSummaries.length > 0) {
|
|
350
|
+
keptStatements.push(omittedSummaries.join('\n'));
|
|
351
|
+
}
|
|
352
|
+
|
|
353
|
+
return keptStatements.join('\n\n');
|
|
354
|
+
}
|
|
355
|
+
|
|
356
|
+
/**
|
|
357
|
+
* Strips full-line YAML comments and collapses blank runs.
|
|
358
|
+
* Lines inside block scalars (| or >) are preserved verbatim, including
|
|
359
|
+
* lines that begin with `#`, since they are string content rather than comments.
|
|
360
|
+
*/
|
|
361
|
+
export function optimizeYAML(yamlCode) {
|
|
362
|
+
if (!yamlCode || !yamlCode.trim()) return yamlCode;
|
|
363
|
+
|
|
364
|
+
const lines = yamlCode.split('\n');
|
|
365
|
+
const kept = [];
|
|
366
|
+
let blankRun = 0;
|
|
367
|
+
let blockScalarIndent = null;
|
|
368
|
+
|
|
369
|
+
for (const line of lines) {
|
|
370
|
+
const content = line.replace(/\s+$/, '');
|
|
371
|
+
const leading = line.length - line.trimStart().length;
|
|
372
|
+
const trimmed = line.trim();
|
|
373
|
+
|
|
374
|
+
if (blockScalarIndent !== null) {
|
|
375
|
+
if (trimmed === '' || leading > blockScalarIndent) {
|
|
376
|
+
kept.push(content);
|
|
377
|
+
continue;
|
|
378
|
+
}
|
|
379
|
+
blockScalarIndent = null;
|
|
380
|
+
}
|
|
381
|
+
|
|
382
|
+
if (/:\s*[|>][+-]?\s*$/.test(line)) {
|
|
383
|
+
blockScalarIndent = leading;
|
|
384
|
+
kept.push(content);
|
|
385
|
+
blankRun = 0;
|
|
386
|
+
continue;
|
|
387
|
+
}
|
|
388
|
+
|
|
389
|
+
if (trimmed.startsWith('#')) continue;
|
|
390
|
+
|
|
391
|
+
if (trimmed === '') {
|
|
392
|
+
blankRun++;
|
|
393
|
+
if (blankRun <= 1) kept.push('');
|
|
394
|
+
continue;
|
|
395
|
+
}
|
|
396
|
+
|
|
397
|
+
blankRun = 0;
|
|
398
|
+
kept.push(content);
|
|
399
|
+
}
|
|
400
|
+
|
|
401
|
+
return kept.join('\n').trim() + '\n';
|
|
402
|
+
}
|
|
403
|
+
|
|
404
|
+
/**
|
|
405
|
+
* Collapses multi-line RUN instructions in Dockerfiles while preserving
|
|
406
|
+
* stage structure (FROM/COPY/CMD/ENTRYPOINT) and all non-RUN directives.
|
|
407
|
+
*/
|
|
408
|
+
export function optimizeDockerfile(dockerCode) {
|
|
409
|
+
if (!dockerCode || !dockerCode.trim()) return dockerCode;
|
|
410
|
+
|
|
411
|
+
const lines = dockerCode.split('\n');
|
|
412
|
+
const out = [];
|
|
413
|
+
let i = 0;
|
|
414
|
+
|
|
415
|
+
while (i < lines.length) {
|
|
416
|
+
const line = lines[i];
|
|
417
|
+
const trimmed = line.trim();
|
|
418
|
+
|
|
419
|
+
if (trimmed === '' || trimmed.startsWith('#')) {
|
|
420
|
+
out.push(line.replace(/\s+$/, ''));
|
|
421
|
+
i++;
|
|
422
|
+
continue;
|
|
423
|
+
}
|
|
424
|
+
|
|
425
|
+
if (/^RUN\b/i.test(trimmed)) {
|
|
426
|
+
const start = i;
|
|
427
|
+
while (i < lines.length && /\\\s*$/.test(lines[i])) i++;
|
|
428
|
+
i++;
|
|
429
|
+
const total = i - start;
|
|
430
|
+
|
|
431
|
+
if (total <= 5) {
|
|
432
|
+
for (let j = start; j < i; j++) out.push(lines[j].replace(/\s+$/, ''));
|
|
433
|
+
} else {
|
|
434
|
+
out.push(lines[start].replace(/\s+$/, ''));
|
|
435
|
+
out.push(lines[start + 1].replace(/\s+$/, ''));
|
|
436
|
+
out.push(`# ...${total - 3} lines omitted from RUN...`);
|
|
437
|
+
out.push(lines[i - 1].replace(/\s+$/, ''));
|
|
438
|
+
}
|
|
439
|
+
continue;
|
|
440
|
+
}
|
|
441
|
+
|
|
442
|
+
out.push(line.replace(/\s+$/, ''));
|
|
443
|
+
i++;
|
|
444
|
+
}
|
|
445
|
+
|
|
446
|
+
return out.join('\n').replace(/\n{3,}/g, '\n\n').trim() + '\n';
|
|
447
|
+
}
|
|
448
|
+
|
|
449
|
+
/**
|
|
450
|
+
* Truncates fenced code blocks longer than 32 lines in Markdown.
|
|
451
|
+
* Headings, prose, tables and short code examples are preserved verbatim.
|
|
452
|
+
*/
|
|
453
|
+
export function optimizeMarkdown(mdCode) {
|
|
454
|
+
if (!mdCode || !mdCode.trim()) return mdCode;
|
|
455
|
+
|
|
456
|
+
const lines = mdCode.split('\n');
|
|
457
|
+
const out = [];
|
|
458
|
+
let fenceChar = null;
|
|
459
|
+
let fenceLen = 0;
|
|
460
|
+
let codeBuffer = [];
|
|
461
|
+
|
|
462
|
+
const flush = (closed) => {
|
|
463
|
+
if (codeBuffer.length > 32) {
|
|
464
|
+
out.push(codeBuffer[0]);
|
|
465
|
+
out.push(`// ...${codeBuffer.length - (closed ? 2 : 1)} lines omitted...`);
|
|
466
|
+
if (closed) out.push(codeBuffer[codeBuffer.length - 1]);
|
|
467
|
+
} else {
|
|
468
|
+
out.push(...codeBuffer);
|
|
469
|
+
}
|
|
470
|
+
codeBuffer = [];
|
|
471
|
+
};
|
|
472
|
+
|
|
473
|
+
for (const line of lines) {
|
|
474
|
+
if (fenceChar === null) {
|
|
475
|
+
const open = line.match(/^\s{0,3}(`{3,}|~{3,})/);
|
|
476
|
+
if (open) {
|
|
477
|
+
fenceChar = open[1][0];
|
|
478
|
+
fenceLen = open[1].length;
|
|
479
|
+
codeBuffer.push(line.replace(/\s+$/, ''));
|
|
480
|
+
} else {
|
|
481
|
+
out.push(line.replace(/\s+$/, ''));
|
|
482
|
+
}
|
|
483
|
+
continue;
|
|
484
|
+
}
|
|
485
|
+
|
|
486
|
+
codeBuffer.push(line.replace(/\s+$/, ''));
|
|
487
|
+
const close = line.match(/^\s{0,3}([`~]{3,})\s*$/);
|
|
488
|
+
if (close && close[1][0] === fenceChar && close[1].length >= fenceLen) {
|
|
489
|
+
flush(true);
|
|
490
|
+
fenceChar = null;
|
|
491
|
+
fenceLen = 0;
|
|
492
|
+
}
|
|
493
|
+
}
|
|
494
|
+
|
|
495
|
+
if (fenceChar !== null) flush(false);
|
|
496
|
+
|
|
497
|
+
return out.join('\n').replace(/\n{3,}/g, '\n\n').trim() + '\n';
|
|
498
|
+
}
|
|
499
|
+
|
|
500
|
+
const PY_CORE_LOGIC_REGEX = /^(_+)?(is|has|can|should|calc|calculate|validate|check|parse|format|sanitize)(_|[A-Z0-9])/i;
|
|
501
|
+
|
|
502
|
+
/**
|
|
503
|
+
* Skeletonizes Python code: preserves functions <= maxPreserveLines or matching core logic whitelist,
|
|
504
|
+
* retains function docstrings, and replaces truncated function bodies with raise NotImplementedError.
|
|
505
|
+
*/
|
|
506
|
+
export function skeletonizePython(code, maxPreserveLines = 8) {
|
|
507
|
+
if (!code || !code.trim()) return code;
|
|
508
|
+
|
|
509
|
+
const lines = code.split('\n');
|
|
510
|
+
const resultLines = [];
|
|
511
|
+
let i = 0;
|
|
512
|
+
|
|
513
|
+
while (i < lines.length) {
|
|
514
|
+
const line = lines[i];
|
|
515
|
+
|
|
516
|
+
// Detect function definition: def func(...) or async def func(...)
|
|
517
|
+
const defMatch = line.match(/^([ \t]*)(?:async\s+)?def\s+([a-zA-Z0-9_]+)\s*\(/);
|
|
518
|
+
if (!defMatch) {
|
|
519
|
+
resultLines.push(line);
|
|
520
|
+
i++;
|
|
521
|
+
continue;
|
|
522
|
+
}
|
|
523
|
+
|
|
524
|
+
const indent = defMatch[1];
|
|
525
|
+
const funcName = defMatch[2];
|
|
526
|
+
const defStartLine = i;
|
|
527
|
+
|
|
528
|
+
// Collect full signature until line ending with ':' with balanced parentheses
|
|
529
|
+
const sigLines = [line];
|
|
530
|
+
let parenDepth = 0;
|
|
531
|
+
for (const char of line.slice(line.indexOf('('))) {
|
|
532
|
+
if (char === '(' || char === '[' || char === '{') parenDepth++;
|
|
533
|
+
else if (char === ')' || char === ']' || char === '}') parenDepth--;
|
|
534
|
+
}
|
|
535
|
+
|
|
536
|
+
let sigEndLine = i;
|
|
537
|
+
while (parenDepth > 0 && sigEndLine + 1 < lines.length) {
|
|
538
|
+
sigEndLine++;
|
|
539
|
+
const nextSigLine = lines[sigEndLine];
|
|
540
|
+
sigLines.push(nextSigLine);
|
|
541
|
+
for (const char of nextSigLine) {
|
|
542
|
+
if (char === '(' || char === '[' || char === '{') parenDepth++;
|
|
543
|
+
else if (char === ')' || char === ']' || char === '}') parenDepth--;
|
|
544
|
+
}
|
|
545
|
+
}
|
|
546
|
+
|
|
547
|
+
// Verify that signature ends with ':' (ignoring comments)
|
|
548
|
+
const lastSigLine = lines[sigEndLine];
|
|
549
|
+
const strippedLast = lastSigLine.replace(/#.*$/, '').trim();
|
|
550
|
+
if (!strippedLast.endsWith(':')) {
|
|
551
|
+
resultLines.push(line);
|
|
552
|
+
i++;
|
|
553
|
+
continue;
|
|
554
|
+
}
|
|
555
|
+
|
|
556
|
+
// Identify body lines (lines following signature with deeper indentation)
|
|
557
|
+
const bodyStart = sigEndLine + 1;
|
|
558
|
+
let bodyEnd = bodyStart - 1;
|
|
559
|
+
|
|
560
|
+
while (bodyEnd + 1 < lines.length) {
|
|
561
|
+
const nextLine = lines[bodyEnd + 1];
|
|
562
|
+
const nextTrimmed = nextLine.trim();
|
|
563
|
+
|
|
564
|
+
if (nextTrimmed === '') {
|
|
565
|
+
bodyEnd++;
|
|
566
|
+
continue;
|
|
567
|
+
}
|
|
568
|
+
|
|
569
|
+
const nextIndent = nextLine.match(/^[ \t]*/)[0];
|
|
570
|
+
if (nextIndent.length > indent.length && nextLine.startsWith(indent)) {
|
|
571
|
+
bodyEnd++;
|
|
572
|
+
} else {
|
|
573
|
+
break;
|
|
574
|
+
}
|
|
575
|
+
}
|
|
576
|
+
|
|
577
|
+
// Strip trailing empty lines from body
|
|
578
|
+
while (bodyEnd >= bodyStart && lines[bodyEnd].trim() === '') {
|
|
579
|
+
bodyEnd--;
|
|
580
|
+
}
|
|
581
|
+
|
|
582
|
+
const bodyLines = bodyEnd >= bodyStart ? lines.slice(bodyStart, bodyEnd + 1) : [];
|
|
583
|
+
const totalBodyLines = bodyLines.length;
|
|
584
|
+
|
|
585
|
+
// Preserve short or whitelisted functions
|
|
586
|
+
if (
|
|
587
|
+
totalBodyLines <= maxPreserveLines ||
|
|
588
|
+
(funcName && PY_CORE_LOGIC_REGEX.test(funcName))
|
|
589
|
+
) {
|
|
590
|
+
for (let k = defStartLine; k <= bodyEnd; k++) {
|
|
591
|
+
resultLines.push(lines[k]);
|
|
592
|
+
}
|
|
593
|
+
i = bodyEnd + 1;
|
|
594
|
+
continue;
|
|
595
|
+
}
|
|
596
|
+
|
|
597
|
+
// Output signature
|
|
598
|
+
for (const sigLine of sigLines) {
|
|
599
|
+
resultLines.push(sigLine);
|
|
600
|
+
}
|
|
601
|
+
|
|
602
|
+
// Determine body indentation
|
|
603
|
+
let bodyIndent = indent + ' ';
|
|
604
|
+
if (bodyLines.length > 0) {
|
|
605
|
+
const firstNonEmpty = bodyLines.find((l) => l.trim() !== '');
|
|
606
|
+
if (firstNonEmpty) {
|
|
607
|
+
bodyIndent = firstNonEmpty.match(/^[ \t]*/)[0];
|
|
608
|
+
}
|
|
609
|
+
}
|
|
610
|
+
|
|
611
|
+
// Preserve docstring if present as first statement
|
|
612
|
+
const docstringLines = [];
|
|
613
|
+
if (bodyLines.length > 0) {
|
|
614
|
+
const firstLine = bodyLines[0].trim();
|
|
615
|
+
const docMatch = firstLine.match(/^(?:r|u|f)?("""|''')/i);
|
|
616
|
+
if (docMatch) {
|
|
617
|
+
const quote = docMatch[1];
|
|
618
|
+
if (firstLine.length > quote.length && firstLine.slice(quote.length).includes(quote)) {
|
|
619
|
+
docstringLines.push(bodyLines[0]);
|
|
620
|
+
} else {
|
|
621
|
+
docstringLines.push(bodyLines[0]);
|
|
622
|
+
for (let d = 1; d < bodyLines.length; d++) {
|
|
623
|
+
docstringLines.push(bodyLines[d]);
|
|
624
|
+
if (bodyLines[d].includes(quote)) {
|
|
625
|
+
break;
|
|
626
|
+
}
|
|
627
|
+
}
|
|
628
|
+
}
|
|
629
|
+
}
|
|
630
|
+
}
|
|
631
|
+
|
|
632
|
+
for (const dLine of docstringLines) {
|
|
633
|
+
resultLines.push(dLine);
|
|
634
|
+
}
|
|
635
|
+
|
|
636
|
+
resultLines.push(
|
|
637
|
+
`${bodyIndent}raise NotImplementedError("Implementation omitted by repomix-semantic-compressor")`
|
|
638
|
+
);
|
|
639
|
+
|
|
640
|
+
i = bodyEnd + 1;
|
|
641
|
+
}
|
|
642
|
+
|
|
643
|
+
return resultLines.join('\n');
|
|
644
|
+
}
|
|
645
|
+
|
|
646
|
+
const GO_CORE_LOGIC_REGEX = /^(is|has|can|should|calc|calculate|validate|check|parse|format|sanitize)[A-Z0-9_]/i;
|
|
647
|
+
|
|
648
|
+
/**
|
|
649
|
+
* Skeletonizes Go code: preserves functions <= maxPreserveLines or matching core logic whitelist,
|
|
650
|
+
* and replaces truncated function bodies with panic("Implementation omitted by repomix-semantic-compressor").
|
|
651
|
+
*/
|
|
652
|
+
export function skeletonizeGo(code, maxPreserveLines = 8) {
|
|
653
|
+
if (!code || !code.trim()) return code;
|
|
654
|
+
|
|
655
|
+
const replacements = [];
|
|
656
|
+
let inLineComment = false;
|
|
657
|
+
let inBlockComment = false;
|
|
658
|
+
let inString = false;
|
|
659
|
+
let inRawString = false;
|
|
660
|
+
let inRune = false;
|
|
661
|
+
|
|
662
|
+
for (let i = 0; i < code.length; i++) {
|
|
663
|
+
const char = code[i];
|
|
664
|
+
const nextChar = code[i + 1];
|
|
665
|
+
|
|
666
|
+
if (inLineComment) {
|
|
667
|
+
if (char === '\n') inLineComment = false;
|
|
668
|
+
continue;
|
|
669
|
+
}
|
|
670
|
+
if (inBlockComment) {
|
|
671
|
+
if (char === '*' && nextChar === '/') {
|
|
672
|
+
inBlockComment = false;
|
|
673
|
+
i++;
|
|
674
|
+
}
|
|
675
|
+
continue;
|
|
676
|
+
}
|
|
677
|
+
if (inString) {
|
|
678
|
+
if (char === '\\') {
|
|
679
|
+
i++;
|
|
680
|
+
} else if (char === '"') {
|
|
681
|
+
inString = false;
|
|
682
|
+
}
|
|
683
|
+
continue;
|
|
684
|
+
}
|
|
685
|
+
if (inRawString) {
|
|
686
|
+
if (char === '`') inRawString = false;
|
|
687
|
+
continue;
|
|
688
|
+
}
|
|
689
|
+
if (inRune) {
|
|
690
|
+
if (char === '\\') {
|
|
691
|
+
i++;
|
|
692
|
+
} else if (char === "'") {
|
|
693
|
+
inRune = false;
|
|
694
|
+
}
|
|
695
|
+
continue;
|
|
696
|
+
}
|
|
697
|
+
|
|
698
|
+
if (char === '/' && nextChar === '/') {
|
|
699
|
+
inLineComment = true;
|
|
700
|
+
i++;
|
|
701
|
+
continue;
|
|
702
|
+
}
|
|
703
|
+
if (char === '/' && nextChar === '*') {
|
|
704
|
+
inBlockComment = true;
|
|
705
|
+
i++;
|
|
706
|
+
continue;
|
|
707
|
+
}
|
|
708
|
+
if (char === '"') {
|
|
709
|
+
inString = true;
|
|
710
|
+
continue;
|
|
711
|
+
}
|
|
712
|
+
if (char === '`') {
|
|
713
|
+
inRawString = true;
|
|
714
|
+
continue;
|
|
715
|
+
}
|
|
716
|
+
if (char === "'") {
|
|
717
|
+
inRune = true;
|
|
718
|
+
continue;
|
|
719
|
+
}
|
|
720
|
+
|
|
721
|
+
const isFuncStart =
|
|
722
|
+
(i === 0 || /[\s;}]/.test(code[i - 1])) &&
|
|
723
|
+
code.startsWith('func', i) &&
|
|
724
|
+
(code.length === i + 4 || /[\s(]/.test(code[i + 4]));
|
|
725
|
+
|
|
726
|
+
if (!isFuncStart) continue;
|
|
727
|
+
|
|
728
|
+
const funcStart = i;
|
|
729
|
+
let scanIdx = i + 4;
|
|
730
|
+
let parenDepth = 0;
|
|
731
|
+
let bodyOpenIdx = -1;
|
|
732
|
+
|
|
733
|
+
while (scanIdx < code.length) {
|
|
734
|
+
const c = code[scanIdx];
|
|
735
|
+
const nc = code[scanIdx + 1];
|
|
736
|
+
|
|
737
|
+
if (c === '/' && nc === '/') {
|
|
738
|
+
scanIdx += 2;
|
|
739
|
+
while (scanIdx < code.length && code[scanIdx] !== '\n') scanIdx++;
|
|
740
|
+
continue;
|
|
741
|
+
}
|
|
742
|
+
if (c === '/' && nc === '*') {
|
|
743
|
+
scanIdx += 2;
|
|
744
|
+
while (scanIdx < code.length && !(code[scanIdx] === '*' && code[scanIdx + 1] === '/')) scanIdx++;
|
|
745
|
+
scanIdx += 2;
|
|
746
|
+
continue;
|
|
747
|
+
}
|
|
748
|
+
if (c === '"') {
|
|
749
|
+
scanIdx++;
|
|
750
|
+
while (scanIdx < code.length && code[scanIdx] !== '"') {
|
|
751
|
+
if (code[scanIdx] === '\\') scanIdx++;
|
|
752
|
+
scanIdx++;
|
|
753
|
+
}
|
|
754
|
+
scanIdx++;
|
|
755
|
+
continue;
|
|
756
|
+
}
|
|
757
|
+
if (c === '`') {
|
|
758
|
+
scanIdx++;
|
|
759
|
+
while (scanIdx < code.length && code[scanIdx] !== '`') scanIdx++;
|
|
760
|
+
scanIdx++;
|
|
761
|
+
continue;
|
|
762
|
+
}
|
|
763
|
+
|
|
764
|
+
if (c === '(') parenDepth++;
|
|
765
|
+
else if (c === ')') parenDepth--;
|
|
766
|
+
|
|
767
|
+
if (parenDepth === 0 && (c === ';' || c === '\n')) {
|
|
768
|
+
break;
|
|
769
|
+
}
|
|
770
|
+
|
|
771
|
+
if (c === '{' && parenDepth === 0) {
|
|
772
|
+
bodyOpenIdx = scanIdx;
|
|
773
|
+
break;
|
|
774
|
+
}
|
|
775
|
+
|
|
776
|
+
scanIdx++;
|
|
777
|
+
}
|
|
778
|
+
|
|
779
|
+
if (bodyOpenIdx === -1) continue;
|
|
780
|
+
|
|
781
|
+
const sig = code.slice(funcStart, bodyOpenIdx);
|
|
782
|
+
const nameMatch = sig.match(/func\s*(?:\([^)]*\)\s*)?([a-zA-Z0-9_]+)/);
|
|
783
|
+
const funcName = nameMatch ? nameMatch[1] : '';
|
|
784
|
+
|
|
785
|
+
let braceDepth = 1;
|
|
786
|
+
let bodyCloseIdx = -1;
|
|
787
|
+
let j = bodyOpenIdx + 1;
|
|
788
|
+
|
|
789
|
+
while (j < code.length) {
|
|
790
|
+
const c = code[j];
|
|
791
|
+
const nc = code[j + 1];
|
|
792
|
+
|
|
793
|
+
if (c === '/' && nc === '/') {
|
|
794
|
+
j += 2;
|
|
795
|
+
while (j < code.length && code[j] !== '\n') j++;
|
|
796
|
+
continue;
|
|
797
|
+
}
|
|
798
|
+
if (c === '/' && nc === '*') {
|
|
799
|
+
j += 2;
|
|
800
|
+
while (j < code.length && !(code[j] === '*' && code[j + 1] === '/')) j++;
|
|
801
|
+
j += 2;
|
|
802
|
+
continue;
|
|
803
|
+
}
|
|
804
|
+
if (c === '"') {
|
|
805
|
+
j++;
|
|
806
|
+
while (j < code.length && code[j] !== '"') {
|
|
807
|
+
if (code[j] === '\\') j++;
|
|
808
|
+
j++;
|
|
809
|
+
}
|
|
810
|
+
j++;
|
|
811
|
+
continue;
|
|
812
|
+
}
|
|
813
|
+
if (c === '`') {
|
|
814
|
+
j++;
|
|
815
|
+
while (j < code.length && code[j] !== '`') j++;
|
|
816
|
+
j++;
|
|
817
|
+
continue;
|
|
818
|
+
}
|
|
819
|
+
if (c === "'") {
|
|
820
|
+
j++;
|
|
821
|
+
while (j < code.length && code[j] !== "'") {
|
|
822
|
+
if (code[j] === '\\') j++;
|
|
823
|
+
j++;
|
|
824
|
+
}
|
|
825
|
+
j++;
|
|
826
|
+
continue;
|
|
827
|
+
}
|
|
828
|
+
|
|
829
|
+
if (c === '{') braceDepth++;
|
|
830
|
+
else if (c === '}') {
|
|
831
|
+
braceDepth--;
|
|
832
|
+
if (braceDepth === 0) {
|
|
833
|
+
bodyCloseIdx = j;
|
|
834
|
+
break;
|
|
835
|
+
}
|
|
836
|
+
}
|
|
837
|
+
j++;
|
|
838
|
+
}
|
|
839
|
+
|
|
840
|
+
if (bodyCloseIdx === -1) continue;
|
|
841
|
+
|
|
842
|
+
const bodyContent = code.slice(bodyOpenIdx + 1, bodyCloseIdx);
|
|
843
|
+
const lineCount = bodyContent.split('\n').length;
|
|
844
|
+
|
|
845
|
+
if (
|
|
846
|
+
lineCount > maxPreserveLines &&
|
|
847
|
+
(!funcName || !GO_CORE_LOGIC_REGEX.test(funcName))
|
|
848
|
+
) {
|
|
849
|
+
const lineStart = code.lastIndexOf('\n', funcStart);
|
|
850
|
+
const indent = code.slice(lineStart + 1, funcStart).match(/^[ \t]*/)[0];
|
|
851
|
+
const tabOrSpace = indent.includes('\t') || indent === '' ? '\t' : ' ';
|
|
852
|
+
|
|
853
|
+
replacements.push({
|
|
854
|
+
start: bodyOpenIdx,
|
|
855
|
+
end: bodyCloseIdx + 1,
|
|
856
|
+
text: `{\n${indent}${tabOrSpace}panic("Implementation omitted by repomix-semantic-compressor")\n${indent}}`
|
|
857
|
+
});
|
|
858
|
+
|
|
859
|
+
i = bodyCloseIdx;
|
|
860
|
+
}
|
|
861
|
+
}
|
|
862
|
+
|
|
863
|
+
replacements.sort((a, b) => b.start - a.start);
|
|
864
|
+
let optimized = code;
|
|
865
|
+
for (const r of replacements) {
|
|
866
|
+
optimized = optimized.slice(0, r.start) + r.text + optimized.slice(r.end);
|
|
867
|
+
}
|
|
868
|
+
|
|
869
|
+
return optimized;
|
|
870
|
+
}
|