crbro-memory 1.0.1 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +121 -122
- package/bin/crbro.js +280 -41
- package/dist/engine/license.d.ts +32 -7
- package/dist/engine/license.d.ts.map +1 -1
- package/dist/engine/license.js +243 -50
- package/dist/engine/license.js.map +1 -1
- package/dist/miner/extractor.d.ts +12 -0
- package/dist/miner/extractor.d.ts.map +1 -0
- package/dist/miner/extractor.js +167 -0
- package/dist/miner/extractor.js.map +1 -0
- package/dist/miner/index.d.ts +60 -0
- package/dist/miner/index.d.ts.map +1 -0
- package/dist/miner/index.js +285 -0
- package/dist/miner/index.js.map +1 -0
- package/dist/miner/scheduler.d.ts +30 -0
- package/dist/miner/scheduler.d.ts.map +1 -0
- package/dist/miner/scheduler.js +293 -0
- package/dist/miner/scheduler.js.map +1 -0
- package/dist/miner/types.d.ts +11 -0
- package/dist/miner/types.d.ts.map +1 -0
- package/dist/miner/types.js +4 -0
- package/dist/miner/types.js.map +1 -0
- package/dist/server.d.ts.map +1 -1
- package/dist/server.js +17 -8
- package/dist/server.js.map +1 -1
- package/package.json +52 -52
package/dist/engine/license.d.ts
CHANGED
|
@@ -2,28 +2,53 @@ import type { Brain } from './brain.js';
|
|
|
2
2
|
import type { LicenseInfo } from '../types/index.js';
|
|
3
3
|
export declare class LicenseEngine {
|
|
4
4
|
private brain;
|
|
5
|
+
private cacheFile;
|
|
6
|
+
private deviceId;
|
|
5
7
|
constructor(brain: Brain);
|
|
6
8
|
/**
|
|
7
|
-
*
|
|
9
|
+
* Check if the license is valid (server-verified + device limit).
|
|
10
|
+
* Uses local cache if verified within 7 days.
|
|
8
11
|
*/
|
|
9
|
-
|
|
12
|
+
isPremium(): Promise<boolean>;
|
|
10
13
|
/**
|
|
11
|
-
*
|
|
14
|
+
* Get the rejection reason (for error messages).
|
|
12
15
|
*/
|
|
13
|
-
|
|
16
|
+
getRejectionReason(): Promise<string | null>;
|
|
14
17
|
/**
|
|
15
18
|
* Get current license info.
|
|
16
19
|
*/
|
|
17
20
|
getLicenseInfo(): Promise<LicenseInfo>;
|
|
18
21
|
/**
|
|
19
|
-
*
|
|
22
|
+
* Legacy compatibility.
|
|
20
23
|
*/
|
|
21
|
-
|
|
24
|
+
canUse(_toolName: string): Promise<boolean>;
|
|
25
|
+
/**
|
|
26
|
+
* Generate a unique device fingerprint from hardware characteristics.
|
|
27
|
+
*/
|
|
28
|
+
private generateDeviceFingerprint;
|
|
22
29
|
/**
|
|
23
30
|
* Resolve the license key — env var takes priority over manifest.
|
|
24
31
|
*/
|
|
25
32
|
private resolveKey;
|
|
26
|
-
|
|
33
|
+
/**
|
|
34
|
+
* Basic format validation — prefix + length + characters.
|
|
35
|
+
*/
|
|
36
|
+
private validateFormat;
|
|
37
|
+
/**
|
|
38
|
+
* Verify license AND register this device.
|
|
39
|
+
* Returns { valid, status } where status explains why if invalid.
|
|
40
|
+
*/
|
|
41
|
+
private verifyAndRegisterDevice;
|
|
42
|
+
/**
|
|
43
|
+
* Fetch license document from Firestore REST API.
|
|
44
|
+
*/
|
|
45
|
+
private fetchLicenseDoc;
|
|
46
|
+
/**
|
|
47
|
+
* Update the devices array in Firestore via REST API PATCH.
|
|
48
|
+
*/
|
|
49
|
+
private updateDevices;
|
|
50
|
+
private readCache;
|
|
51
|
+
private writeCache;
|
|
27
52
|
private getInfo;
|
|
28
53
|
}
|
|
29
54
|
//# sourceMappingURL=license.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"license.d.ts","sourceRoot":"","sources":["../../src/engine/license.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"license.d.ts","sourceRoot":"","sources":["../../src/engine/license.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AACxC,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAmCrD,qBAAa,aAAa;IAIZ,OAAO,CAAC,KAAK;IAHzB,OAAO,CAAC,SAAS,CAAS;IAC1B,OAAO,CAAC,QAAQ,CAAS;gBAEL,KAAK,EAAE,KAAK;IAKhC;;;OAGG;IACG,SAAS,IAAI,OAAO,CAAC,OAAO,CAAC;IA+BnC;;OAEG;IACG,kBAAkB,IAAI,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC;IAYlD;;OAEG;IACG,cAAc,IAAI,OAAO,CAAC,WAAW,CAAC;IAM5C;;OAEG;IACG,MAAM,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IAMjD;;OAEG;IACH,OAAO,CAAC,yBAAyB;IAYjC;;OAEG;YACW,UAAU;IAYxB;;OAEG;IACH,OAAO,CAAC,cAAc;IAStB;;;OAGG;YACW,uBAAuB;IAsCrC;;OAEG;IACH,OAAO,CAAC,eAAe;IA0BvB;;OAEG;IACH,OAAO,CAAC,aAAa;YA6CP,SAAS;YAUT,UAAU;IAQxB,OAAO,CAAC,OAAO;CAUhB"}
|
package/dist/engine/license.js
CHANGED
|
@@ -1,101 +1,294 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
// ─── CRBRO License Engine ────────────────────────────────────────
|
|
3
|
-
//
|
|
3
|
+
// Server-side license validation against Firestore + device limit + local cache
|
|
4
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
5
|
+
if (k2 === undefined) k2 = k;
|
|
6
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
7
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
8
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
9
|
+
}
|
|
10
|
+
Object.defineProperty(o, k2, desc);
|
|
11
|
+
}) : (function(o, m, k, k2) {
|
|
12
|
+
if (k2 === undefined) k2 = k;
|
|
13
|
+
o[k2] = m[k];
|
|
14
|
+
}));
|
|
15
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
16
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
17
|
+
}) : function(o, v) {
|
|
18
|
+
o["default"] = v;
|
|
19
|
+
});
|
|
20
|
+
var __importStar = (this && this.__importStar) || (function () {
|
|
21
|
+
var ownKeys = function(o) {
|
|
22
|
+
ownKeys = Object.getOwnPropertyNames || function (o) {
|
|
23
|
+
var ar = [];
|
|
24
|
+
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
|
25
|
+
return ar;
|
|
26
|
+
};
|
|
27
|
+
return ownKeys(o);
|
|
28
|
+
};
|
|
29
|
+
return function (mod) {
|
|
30
|
+
if (mod && mod.__esModule) return mod;
|
|
31
|
+
var result = {};
|
|
32
|
+
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
|
33
|
+
__setModuleDefault(result, mod);
|
|
34
|
+
return result;
|
|
35
|
+
};
|
|
36
|
+
})();
|
|
4
37
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
5
38
|
exports.LicenseEngine = void 0;
|
|
6
|
-
|
|
7
|
-
const
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
'crbro_sessions',
|
|
18
|
-
'crbro_context',
|
|
19
|
-
'crbro_hot_topics',
|
|
20
|
-
'crbro_consolidate',
|
|
21
|
-
]);
|
|
22
|
-
// Premium features requiring license
|
|
23
|
-
const PREMIUM_FEATURES = new Set([
|
|
24
|
-
'crbro_global_map',
|
|
25
|
-
'crbro_maintenance',
|
|
26
|
-
]);
|
|
39
|
+
const https = __importStar(require("https"));
|
|
40
|
+
const path = __importStar(require("path"));
|
|
41
|
+
const fs = __importStar(require("fs"));
|
|
42
|
+
const os = __importStar(require("os"));
|
|
43
|
+
const crypto = __importStar(require("crypto"));
|
|
44
|
+
// Firestore project for license validation
|
|
45
|
+
const FIRESTORE_PROJECT = 'synthetica-decks';
|
|
46
|
+
const COLLECTION = 'license-checks';
|
|
47
|
+
const MAX_DEVICES = 3;
|
|
48
|
+
// Cache duration: 7 days in milliseconds
|
|
49
|
+
const CACHE_TTL_MS = 7 * 24 * 60 * 60 * 1000;
|
|
27
50
|
// Valid license key prefix — Synthetica Zero Deck keys
|
|
28
51
|
const LICENSE_PREFIX = 'SYNTH-ZERO-';
|
|
29
52
|
class LicenseEngine {
|
|
30
53
|
brain;
|
|
54
|
+
cacheFile;
|
|
55
|
+
deviceId;
|
|
31
56
|
constructor(brain) {
|
|
32
57
|
this.brain = brain;
|
|
58
|
+
this.cacheFile = path.join(brain.paths.root, '.license-cache.json');
|
|
59
|
+
this.deviceId = this.generateDeviceFingerprint();
|
|
33
60
|
}
|
|
34
61
|
/**
|
|
35
|
-
*
|
|
62
|
+
* Check if the license is valid (server-verified + device limit).
|
|
63
|
+
* Uses local cache if verified within 7 days.
|
|
36
64
|
*/
|
|
37
|
-
async
|
|
38
|
-
const
|
|
39
|
-
if (
|
|
40
|
-
|
|
65
|
+
async isPremium() {
|
|
66
|
+
const key = await this.resolveKey();
|
|
67
|
+
if (!key)
|
|
68
|
+
return false;
|
|
69
|
+
// Quick format check before wasting a network call
|
|
70
|
+
if (!this.validateFormat(key))
|
|
71
|
+
return false;
|
|
72
|
+
// Check local cache first
|
|
73
|
+
const cached = await this.readCache();
|
|
74
|
+
if (cached && cached.key === key && cached.valid && cached.deviceId === this.deviceId) {
|
|
75
|
+
const age = Date.now() - cached.verifiedAt;
|
|
76
|
+
if (age < CACHE_TTL_MS) {
|
|
77
|
+
return true;
|
|
78
|
+
}
|
|
41
79
|
}
|
|
42
|
-
|
|
80
|
+
// Cache expired or missing — verify against Firestore
|
|
81
|
+
const result = await this.verifyAndRegisterDevice(key);
|
|
82
|
+
// Save result to cache
|
|
83
|
+
await this.writeCache({
|
|
84
|
+
key,
|
|
85
|
+
valid: result.valid,
|
|
86
|
+
verifiedAt: Date.now(),
|
|
87
|
+
deviceId: this.deviceId,
|
|
88
|
+
status: result.status,
|
|
89
|
+
});
|
|
90
|
+
return result.valid;
|
|
43
91
|
}
|
|
44
92
|
/**
|
|
45
|
-
*
|
|
93
|
+
* Get the rejection reason (for error messages).
|
|
46
94
|
*/
|
|
47
|
-
async
|
|
48
|
-
if (FREE_FEATURES.has(toolName))
|
|
49
|
-
return true;
|
|
50
|
-
if (!PREMIUM_FEATURES.has(toolName))
|
|
51
|
-
return true; // Unknown tools default to allowed
|
|
95
|
+
async getRejectionReason() {
|
|
52
96
|
const key = await this.resolveKey();
|
|
53
|
-
|
|
97
|
+
if (!key)
|
|
98
|
+
return 'no_key';
|
|
99
|
+
if (!this.validateFormat(key))
|
|
100
|
+
return 'invalid_format';
|
|
101
|
+
const cached = await this.readCache();
|
|
102
|
+
if (cached && cached.key === key && !cached.valid) {
|
|
103
|
+
return cached.status;
|
|
104
|
+
}
|
|
105
|
+
return null;
|
|
54
106
|
}
|
|
55
107
|
/**
|
|
56
108
|
* Get current license info.
|
|
57
109
|
*/
|
|
58
110
|
async getLicenseInfo() {
|
|
59
111
|
const key = await this.resolveKey();
|
|
60
|
-
|
|
112
|
+
const valid = await this.isPremium();
|
|
113
|
+
return this.getInfo(valid ? key : null);
|
|
61
114
|
}
|
|
62
115
|
/**
|
|
63
|
-
*
|
|
116
|
+
* Legacy compatibility.
|
|
64
117
|
*/
|
|
65
|
-
async
|
|
66
|
-
|
|
67
|
-
return this.validateKey(key);
|
|
118
|
+
async canUse(_toolName) {
|
|
119
|
+
return this.isPremium();
|
|
68
120
|
}
|
|
69
121
|
// ─── Private ─────────────────────────────────────────────────
|
|
122
|
+
/**
|
|
123
|
+
* Generate a unique device fingerprint from hardware characteristics.
|
|
124
|
+
*/
|
|
125
|
+
generateDeviceFingerprint() {
|
|
126
|
+
const raw = [
|
|
127
|
+
os.hostname(),
|
|
128
|
+
os.userInfo().username,
|
|
129
|
+
os.platform(),
|
|
130
|
+
os.arch(),
|
|
131
|
+
os.cpus()[0]?.model || 'unknown',
|
|
132
|
+
].join('|');
|
|
133
|
+
return crypto.createHash('sha256').update(raw).digest('hex').substring(0, 16);
|
|
134
|
+
}
|
|
70
135
|
/**
|
|
71
136
|
* Resolve the license key — env var takes priority over manifest.
|
|
72
137
|
*/
|
|
73
138
|
async resolveKey() {
|
|
74
|
-
// 1. Check environment variable first (set in mcp_config.json env block)
|
|
75
139
|
const envKey = process.env.CRBRO_LICENSE_KEY || null;
|
|
76
|
-
if (envKey && this.
|
|
140
|
+
if (envKey && this.validateFormat(envKey))
|
|
77
141
|
return envKey;
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
142
|
+
try {
|
|
143
|
+
const manifest = await this.brain.getManifest();
|
|
144
|
+
return manifest.license_key || null;
|
|
145
|
+
}
|
|
146
|
+
catch {
|
|
147
|
+
return null;
|
|
148
|
+
}
|
|
81
149
|
}
|
|
82
|
-
|
|
150
|
+
/**
|
|
151
|
+
* Basic format validation — prefix + length + characters.
|
|
152
|
+
*/
|
|
153
|
+
validateFormat(key) {
|
|
83
154
|
if (!key)
|
|
84
155
|
return false;
|
|
85
|
-
// Basic validation — prefix check + length
|
|
86
156
|
if (!key.startsWith(LICENSE_PREFIX))
|
|
87
157
|
return false;
|
|
88
158
|
if (key.length < 20)
|
|
89
159
|
return false;
|
|
90
|
-
// Checksum validation (simple)
|
|
91
160
|
const payload = key.substring(LICENSE_PREFIX.length);
|
|
92
|
-
// Must be alphanumeric + hyphens
|
|
93
161
|
if (!/^[A-Z0-9-]+$/.test(payload))
|
|
94
162
|
return false;
|
|
95
163
|
return true;
|
|
96
164
|
}
|
|
165
|
+
/**
|
|
166
|
+
* Verify license AND register this device.
|
|
167
|
+
* Returns { valid, status } where status explains why if invalid.
|
|
168
|
+
*/
|
|
169
|
+
async verifyAndRegisterDevice(key) {
|
|
170
|
+
// Step 1: Fetch the license document
|
|
171
|
+
const doc = await this.fetchLicenseDoc(key);
|
|
172
|
+
if (!doc) {
|
|
173
|
+
return { valid: false, status: 'key_not_found' };
|
|
174
|
+
}
|
|
175
|
+
const status = doc.fields?.status?.stringValue;
|
|
176
|
+
if (status !== 'active') {
|
|
177
|
+
return { valid: false, status: 'license_revoked' };
|
|
178
|
+
}
|
|
179
|
+
// Step 2: Check device limit
|
|
180
|
+
const devicesArray = doc.fields?.devices?.arrayValue?.values || [];
|
|
181
|
+
const registeredDevices = devicesArray.map(v => v.stringValue || '');
|
|
182
|
+
// This device is already registered — all good
|
|
183
|
+
if (registeredDevices.includes(this.deviceId)) {
|
|
184
|
+
return { valid: true, status: 'active' };
|
|
185
|
+
}
|
|
186
|
+
// Check if there's room for another device
|
|
187
|
+
if (registeredDevices.length >= MAX_DEVICES) {
|
|
188
|
+
return { valid: false, status: 'device_limit_exceeded' };
|
|
189
|
+
}
|
|
190
|
+
// Step 3: Register this device
|
|
191
|
+
const newDevices = [...registeredDevices, this.deviceId];
|
|
192
|
+
const registered = await this.updateDevices(key, newDevices);
|
|
193
|
+
if (registered) {
|
|
194
|
+
return { valid: true, status: 'active' };
|
|
195
|
+
}
|
|
196
|
+
else {
|
|
197
|
+
// Registration failed (possibly race condition or rule violation)
|
|
198
|
+
return { valid: false, status: 'device_registration_failed' };
|
|
199
|
+
}
|
|
200
|
+
}
|
|
201
|
+
/**
|
|
202
|
+
* Fetch license document from Firestore REST API.
|
|
203
|
+
*/
|
|
204
|
+
fetchLicenseDoc(key) {
|
|
205
|
+
return new Promise((resolve) => {
|
|
206
|
+
const encodedKey = encodeURIComponent(key);
|
|
207
|
+
const url = `https://firestore.googleapis.com/v1/projects/${FIRESTORE_PROJECT}/databases/(default)/documents/${COLLECTION}/${encodedKey}`;
|
|
208
|
+
const req = https.get(url, { timeout: 10000 }, (res) => {
|
|
209
|
+
let data = '';
|
|
210
|
+
res.on('data', (chunk) => { data += chunk.toString(); });
|
|
211
|
+
res.on('end', () => {
|
|
212
|
+
try {
|
|
213
|
+
if (res.statusCode === 200) {
|
|
214
|
+
resolve(JSON.parse(data));
|
|
215
|
+
}
|
|
216
|
+
else {
|
|
217
|
+
resolve(null);
|
|
218
|
+
}
|
|
219
|
+
}
|
|
220
|
+
catch {
|
|
221
|
+
resolve(null);
|
|
222
|
+
}
|
|
223
|
+
});
|
|
224
|
+
});
|
|
225
|
+
req.on('error', () => resolve(null));
|
|
226
|
+
req.on('timeout', () => { req.destroy(); resolve(null); });
|
|
227
|
+
});
|
|
228
|
+
}
|
|
229
|
+
/**
|
|
230
|
+
* Update the devices array in Firestore via REST API PATCH.
|
|
231
|
+
*/
|
|
232
|
+
updateDevices(key, devices) {
|
|
233
|
+
return new Promise((resolve) => {
|
|
234
|
+
const encodedKey = encodeURIComponent(key);
|
|
235
|
+
const url = `https://firestore.googleapis.com/v1/projects/${FIRESTORE_PROJECT}/databases/(default)/documents/${COLLECTION}/${encodedKey}?updateMask.fieldPaths=devices`;
|
|
236
|
+
const body = JSON.stringify({
|
|
237
|
+
fields: {
|
|
238
|
+
devices: {
|
|
239
|
+
arrayValue: {
|
|
240
|
+
values: devices.map(d => ({ stringValue: d })),
|
|
241
|
+
},
|
|
242
|
+
},
|
|
243
|
+
},
|
|
244
|
+
});
|
|
245
|
+
const parsed = new URL(url);
|
|
246
|
+
const options = {
|
|
247
|
+
hostname: parsed.hostname,
|
|
248
|
+
path: parsed.pathname + parsed.search,
|
|
249
|
+
method: 'PATCH',
|
|
250
|
+
timeout: 10000,
|
|
251
|
+
headers: {
|
|
252
|
+
'Content-Type': 'application/json',
|
|
253
|
+
'Content-Length': Buffer.byteLength(body),
|
|
254
|
+
},
|
|
255
|
+
};
|
|
256
|
+
const req = https.request(options, (res) => {
|
|
257
|
+
let data = '';
|
|
258
|
+
res.on('data', (chunk) => { data += chunk.toString(); });
|
|
259
|
+
res.on('end', () => {
|
|
260
|
+
// 200 = success, 403 = device limit exceeded (security rule violation)
|
|
261
|
+
resolve(res.statusCode === 200);
|
|
262
|
+
});
|
|
263
|
+
});
|
|
264
|
+
req.on('error', () => resolve(false));
|
|
265
|
+
req.on('timeout', () => { req.destroy(); resolve(false); });
|
|
266
|
+
req.write(body);
|
|
267
|
+
req.end();
|
|
268
|
+
});
|
|
269
|
+
}
|
|
270
|
+
// ─── Cache management ──────────────────────────────────────────
|
|
271
|
+
async readCache() {
|
|
272
|
+
try {
|
|
273
|
+
if (!fs.existsSync(this.cacheFile))
|
|
274
|
+
return null;
|
|
275
|
+
const raw = fs.readFileSync(this.cacheFile, 'utf-8');
|
|
276
|
+
return JSON.parse(raw);
|
|
277
|
+
}
|
|
278
|
+
catch {
|
|
279
|
+
return null;
|
|
280
|
+
}
|
|
281
|
+
}
|
|
282
|
+
async writeCache(cache) {
|
|
283
|
+
try {
|
|
284
|
+
fs.writeFileSync(this.cacheFile, JSON.stringify(cache, null, 2), 'utf-8');
|
|
285
|
+
}
|
|
286
|
+
catch {
|
|
287
|
+
// Silent fail
|
|
288
|
+
}
|
|
289
|
+
}
|
|
97
290
|
getInfo(key) {
|
|
98
|
-
const valid =
|
|
291
|
+
const valid = !!key;
|
|
99
292
|
return {
|
|
100
293
|
valid,
|
|
101
294
|
tier: valid ? 'premium' : 'free',
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"license.js","sourceRoot":"","sources":["../../src/engine/license.ts"],"names":[],"mappings":";AAAA,oEAAoE;AACpE,
|
|
1
|
+
{"version":3,"file":"license.js","sourceRoot":"","sources":["../../src/engine/license.ts"],"names":[],"mappings":";AAAA,oEAAoE;AACpE,gFAAgF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAIhF,6CAA+B;AAC/B,2CAA6B;AAC7B,uCAAyB;AACzB,uCAAyB;AACzB,+CAAiC;AAEjC,2CAA2C;AAC3C,MAAM,iBAAiB,GAAG,kBAAkB,CAAC;AAC7C,MAAM,UAAU,GAAG,gBAAgB,CAAC;AACpC,MAAM,WAAW,GAAG,CAAC,CAAC;AAEtB,yCAAyC;AACzC,MAAM,YAAY,GAAG,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC;AAE7C,uDAAuD;AACvD,MAAM,cAAc,GAAG,aAAa,CAAC;AAmBrC,MAAa,aAAa;IAIJ;IAHZ,SAAS,CAAS;IAClB,QAAQ,CAAS;IAEzB,YAAoB,KAAY;QAAZ,UAAK,GAAL,KAAK,CAAO;QAC9B,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,EAAE,qBAAqB,CAAC,CAAC;QACpE,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,yBAAyB,EAAE,CAAC;IACnD,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,SAAS;QACb,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,UAAU,EAAE,CAAC;QACpC,IAAI,CAAC,GAAG;YAAE,OAAO,KAAK,CAAC;QAEvB,mDAAmD;QACnD,IAAI,CAAC,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC;YAAE,OAAO,KAAK,CAAC;QAE5C,0BAA0B;QAC1B,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,SAAS,EAAE,CAAC;QACtC,IAAI,MAAM,IAAI,MAAM,CAAC,GAAG,KAAK,GAAG,IAAI,MAAM,CAAC,KAAK,IAAI,MAAM,CAAC,QAAQ,KAAK,IAAI,CAAC,QAAQ,EAAE,CAAC;YACtF,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,MAAM,CAAC,UAAU,CAAC;YAC3C,IAAI,GAAG,GAAG,YAAY,EAAE,CAAC;gBACvB,OAAO,IAAI,CAAC;YACd,CAAC;QACH,CAAC;QAED,sDAAsD;QACtD,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,uBAAuB,CAAC,GAAG,CAAC,CAAC;QAEvD,uBAAuB;QACvB,MAAM,IAAI,CAAC,UAAU,CAAC;YACpB,GAAG;YACH,KAAK,EAAE,MAAM,CAAC,KAAK;YACnB,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE;YACtB,QAAQ,EAAE,IAAI,CAAC,QAAQ;YACvB,MAAM,EAAE,MAAM,CAAC,MAAM;SACtB,CAAC,CAAC;QAEH,OAAO,MAAM,CAAC,KAAK,CAAC;IACtB,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,kBAAkB;QACtB,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,UAAU,EAAE,CAAC;QACpC,IAAI,CAAC,GAAG;YAAE,OAAO,QAAQ,CAAC;QAC1B,IAAI,CAAC,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC;YAAE,OAAO,gBAAgB,CAAC;QAEvD,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,SAAS,EAAE,CAAC;QACtC,IAAI,MAAM,IAAI,MAAM,CAAC,GAAG,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;YAClD,OAAO,MAAM,CAAC,MAAM,CAAC;QACvB,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,cAAc;QAClB,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,UAAU,EAAE,CAAC;QACpC,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,SAAS,EAAE,CAAC;QACrC,OAAO,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IAC1C,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,MAAM,CAAC,SAAiB;QAC5B,OAAO,IAAI,CAAC,SAAS,EAAE,CAAC;IAC1B,CAAC;IAED,gEAAgE;IAEhE;;OAEG;IACK,yBAAyB;QAC/B,MAAM,GAAG,GAAG;YACV,EAAE,CAAC,QAAQ,EAAE;YACb,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ;YACtB,EAAE,CAAC,QAAQ,EAAE;YACb,EAAE,CAAC,IAAI,EAAE;YACT,EAAE,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,IAAI,SAAS;SACjC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAEZ,OAAO,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,SAAS,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IAChF,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,UAAU;QACtB,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,iBAAiB,IAAI,IAAI,CAAC;QACrD,IAAI,MAAM,IAAI,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC;YAAE,OAAO,MAAM,CAAC;QAEzD,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,WAAW,EAAE,CAAC;YAChD,OAAO,QAAQ,CAAC,WAAW,IAAI,IAAI,CAAC;QACtC,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;IAED;;OAEG;IACK,cAAc,CAAC,GAAkB;QACvC,IAAI,CAAC,GAAG;YAAE,OAAO,KAAK,CAAC;QACvB,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,cAAc,CAAC;YAAE,OAAO,KAAK,CAAC;QAClD,IAAI,GAAG,CAAC,MAAM,GAAG,EAAE;YAAE,OAAO,KAAK,CAAC;QAClC,MAAM,OAAO,GAAG,GAAG,CAAC,SAAS,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC;QACrD,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,OAAO,CAAC;YAAE,OAAO,KAAK,CAAC;QAChD,OAAO,IAAI,CAAC;IACd,CAAC;IAED;;;OAGG;IACK,KAAK,CAAC,uBAAuB,CAAC,GAAW;QAC/C,qCAAqC;QACrC,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC;QAC5C,IAAI,CAAC,GAAG,EAAE,CAAC;YACT,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,eAAe,EAAE,CAAC;QACnD,CAAC;QAED,MAAM,MAAM,GAAG,GAAG,CAAC,MAAM,EAAE,MAAM,EAAE,WAAW,CAAC;QAC/C,IAAI,MAAM,KAAK,QAAQ,EAAE,CAAC;YACxB,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,iBAAiB,EAAE,CAAC;QACrD,CAAC;QAED,6BAA6B;QAC7B,MAAM,YAAY,GAAG,GAAG,CAAC,MAAM,EAAE,OAAO,EAAE,UAAU,EAAE,MAAM,IAAI,EAAE,CAAC;QACnE,MAAM,iBAAiB,GAAG,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,WAAW,IAAI,EAAE,CAAC,CAAC;QAErE,+CAA+C;QAC/C,IAAI,iBAAiB,CAAC,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC;YAC9C,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC;QAC3C,CAAC;QAED,2CAA2C;QAC3C,IAAI,iBAAiB,CAAC,MAAM,IAAI,WAAW,EAAE,CAAC;YAC5C,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,uBAAuB,EAAE,CAAC;QAC3D,CAAC;QAED,+BAA+B;QAC/B,MAAM,UAAU,GAAG,CAAC,GAAG,iBAAiB,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC;QACzD,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,GAAG,EAAE,UAAU,CAAC,CAAC;QAE7D,IAAI,UAAU,EAAE,CAAC;YACf,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC;QAC3C,CAAC;aAAM,CAAC;YACN,kEAAkE;YAClE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,4BAA4B,EAAE,CAAC;QAChE,CAAC;IACH,CAAC;IAED;;OAEG;IACK,eAAe,CAAC,GAAW;QACjC,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE;YAC7B,MAAM,UAAU,GAAG,kBAAkB,CAAC,GAAG,CAAC,CAAC;YAC3C,MAAM,GAAG,GAAG,gDAAgD,iBAAiB,kCAAkC,UAAU,IAAI,UAAU,EAAE,CAAC;YAE1I,MAAM,GAAG,GAAG,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,EAAE,CAAC,GAAG,EAAE,EAAE;gBACrD,IAAI,IAAI,GAAG,EAAE,CAAC;gBACd,GAAG,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,KAAa,EAAE,EAAE,GAAG,IAAI,IAAI,KAAK,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;gBACjE,GAAG,CAAC,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE;oBACjB,IAAI,CAAC;wBACH,IAAI,GAAG,CAAC,UAAU,KAAK,GAAG,EAAE,CAAC;4BAC3B,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAiB,CAAC,CAAC;wBAC5C,CAAC;6BAAM,CAAC;4BACN,OAAO,CAAC,IAAI,CAAC,CAAC;wBAChB,CAAC;oBACH,CAAC;oBAAC,MAAM,CAAC;wBACP,OAAO,CAAC,IAAI,CAAC,CAAC;oBAChB,CAAC;gBACH,CAAC,CAAC,CAAC;YACL,CAAC,CAAC,CAAC;YAEH,GAAG,CAAC,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC;YACrC,GAAG,CAAC,EAAE,CAAC,SAAS,EAAE,GAAG,EAAE,GAAG,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAC7D,CAAC,CAAC,CAAC;IACL,CAAC;IAED;;OAEG;IACK,aAAa,CAAC,GAAW,EAAE,OAAiB;QAClD,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE;YAC7B,MAAM,UAAU,GAAG,kBAAkB,CAAC,GAAG,CAAC,CAAC;YAC3C,MAAM,GAAG,GAAG,gDAAgD,iBAAiB,kCAAkC,UAAU,IAAI,UAAU,gCAAgC,CAAC;YAExK,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC;gBAC1B,MAAM,EAAE;oBACN,OAAO,EAAE;wBACP,UAAU,EAAE;4BACV,MAAM,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,WAAW,EAAE,CAAC,EAAE,CAAC,CAAC;yBAC/C;qBACF;iBACF;aACF,CAAC,CAAC;YAEH,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;YAC5B,MAAM,OAAO,GAAG;gBACd,QAAQ,EAAE,MAAM,CAAC,QAAQ;gBACzB,IAAI,EAAE,MAAM,CAAC,QAAQ,GAAG,MAAM,CAAC,MAAM;gBACrC,MAAM,EAAE,OAAO;gBACf,OAAO,EAAE,KAAK;gBACd,OAAO,EAAE;oBACP,cAAc,EAAE,kBAAkB;oBAClC,gBAAgB,EAAE,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC;iBAC1C;aACF,CAAC;YAEF,MAAM,GAAG,GAAG,KAAK,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE;gBACzC,IAAI,IAAI,GAAG,EAAE,CAAC;gBACd,GAAG,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,KAAa,EAAE,EAAE,GAAG,IAAI,IAAI,KAAK,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;gBACjE,GAAG,CAAC,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE;oBACjB,uEAAuE;oBACvE,OAAO,CAAC,GAAG,CAAC,UAAU,KAAK,GAAG,CAAC,CAAC;gBAClC,CAAC,CAAC,CAAC;YACL,CAAC,CAAC,CAAC;YAEH,GAAG,CAAC,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC;YACtC,GAAG,CAAC,EAAE,CAAC,SAAS,EAAE,GAAG,EAAE,GAAG,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YAC5D,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;YAChB,GAAG,CAAC,GAAG,EAAE,CAAC;QACZ,CAAC,CAAC,CAAC;IACL,CAAC;IAED,kEAAkE;IAE1D,KAAK,CAAC,SAAS;QACrB,IAAI,CAAC;YACH,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,SAAS,CAAC;gBAAE,OAAO,IAAI,CAAC;YAChD,MAAM,GAAG,GAAG,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;YACrD,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CAAiB,CAAC;QACzC,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,UAAU,CAAC,KAAmB;QAC1C,IAAI,CAAC;YACH,EAAE,CAAC,aAAa,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC;QAC5E,CAAC;QAAC,MAAM,CAAC;YACP,cAAc;QAChB,CAAC;IACH,CAAC;IAEO,OAAO,CAAC,GAAkB;QAChC,MAAM,KAAK,GAAG,CAAC,CAAC,GAAG,CAAC;QACpB,OAAO;YACL,KAAK;YACL,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,MAAM;YAChC,QAAQ,EAAE,KAAK;gBACb,CAAC,CAAC,CAAC,YAAY,EAAE,aAAa,EAAE,QAAQ,EAAE,QAAQ,EAAE,iBAAiB,CAAC;gBACtE,CAAC,CAAC,EAAE;SACP,CAAC;IACJ,CAAC;CACF;AA1QD,sCA0QC"}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
export interface ExtractedContent {
|
|
2
|
+
topics: string[];
|
|
3
|
+
facts: string[];
|
|
4
|
+
decisions: string[];
|
|
5
|
+
technologies: string[];
|
|
6
|
+
summary: string;
|
|
7
|
+
}
|
|
8
|
+
/**
|
|
9
|
+
* Extract structured content from a markdown file's text.
|
|
10
|
+
*/
|
|
11
|
+
export declare function extractContent(text: string, filename: string): ExtractedContent;
|
|
12
|
+
//# sourceMappingURL=extractor.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"extractor.d.ts","sourceRoot":"","sources":["../../src/miner/extractor.ts"],"names":[],"mappings":"AAIA,MAAM,WAAW,gBAAgB;IAC/B,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,KAAK,EAAE,MAAM,EAAE,CAAC;IAChB,SAAS,EAAE,MAAM,EAAE,CAAC;IACpB,YAAY,EAAE,MAAM,EAAE,CAAC;IACvB,OAAO,EAAE,MAAM,CAAC;CACjB;AAiDD;;GAEG;AACH,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,gBAAgB,CAqE/E"}
|
|
@@ -0,0 +1,167 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
// ─── CRBRO Miner: Text Extractor ──────────────────────────────
|
|
3
|
+
// Simple keyword/fact/decision extraction from markdown files.
|
|
4
|
+
// No AI required — uses regex patterns and basic NLP.
|
|
5
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
exports.extractContent = extractContent;
|
|
7
|
+
// Common technology keywords to detect
|
|
8
|
+
const TECH_KEYWORDS = new Set([
|
|
9
|
+
'react', 'vue', 'angular', 'svelte', 'next.js', 'nextjs', 'nuxt', 'vite',
|
|
10
|
+
'node', 'nodejs', 'deno', 'bun', 'express', 'fastify', 'koa',
|
|
11
|
+
'python', 'django', 'flask', 'fastapi',
|
|
12
|
+
'typescript', 'javascript', 'php', 'ruby', 'go', 'rust', 'java', 'kotlin', 'swift',
|
|
13
|
+
'firebase', 'supabase', 'mongodb', 'postgresql', 'mysql', 'redis', 'sqlite',
|
|
14
|
+
'docker', 'kubernetes', 'terraform', 'ansible',
|
|
15
|
+
'aws', 'gcp', 'azure', 'cloudflare', 'vercel', 'netlify',
|
|
16
|
+
'github', 'gitlab', 'bitbucket',
|
|
17
|
+
'wordpress', 'shopify', 'strapi', 'sanity',
|
|
18
|
+
'tailwind', 'css', 'sass', 'less',
|
|
19
|
+
'graphql', 'rest', 'grpc', 'websocket',
|
|
20
|
+
'stripe', 'paypal', 'openai', 'gemini', 'claude', 'ollama',
|
|
21
|
+
'electron', 'tauri', 'flutter', 'react native',
|
|
22
|
+
'cloud run', 'cloud functions', 'lambda', 'ec2', 's3',
|
|
23
|
+
'firestore', 'realtime database', 'cloud storage',
|
|
24
|
+
'seo', 'rank math', 'yoast', 'schema markup',
|
|
25
|
+
'mcp', 'model context protocol',
|
|
26
|
+
]);
|
|
27
|
+
// Decision indicator patterns (multilingual)
|
|
28
|
+
const DECISION_PATTERNS = [
|
|
29
|
+
// English
|
|
30
|
+
/(?:decided|chose|selected|opted|switched|migrated|moved|changed|replaced|upgraded|downgraded)\s+(?:to|from|for)\s+(.+)/gi,
|
|
31
|
+
/(?:we(?:'ll| will)|going to|plan to|need to)\s+(migrate|switch|move|change|replace|implement|build|create|use)\s+(.+)/gi,
|
|
32
|
+
// Spanish
|
|
33
|
+
/(?:decidimos|elegimos|optamos|migramos|cambiamos|reemplazamos)\s+(.+)/gi,
|
|
34
|
+
/(?:vamos a|hay que|necesitamos)\s+(migrar|cambiar|reemplazar|implementar|crear|usar)\s+(.+)/gi,
|
|
35
|
+
// French
|
|
36
|
+
/(?:décidé|choisi|opté|migré|changé|remplacé)\s+(?:de|pour|à)\s+(.+)/gi,
|
|
37
|
+
// German
|
|
38
|
+
/(?:entschieden|gewählt|migriert|gewechselt|ersetzt)\s+(.+)/gi,
|
|
39
|
+
// Italian
|
|
40
|
+
/(?:deciso|scelto|migrato|cambiato|sostituito)\s+(?:di|per|a)\s+(.+)/gi,
|
|
41
|
+
];
|
|
42
|
+
// Fact indicator patterns
|
|
43
|
+
const FACT_PATTERNS = [
|
|
44
|
+
/(?:uses?|utilizes?|runs? on|built with|powered by|depends? on|requires?)\s+(.+)/gi,
|
|
45
|
+
/(?:usa|utiliza|funciona con|construido con|depende de|requiere)\s+(.+)/gi,
|
|
46
|
+
/(?:deployed?|hosted?|served?)\s+(?:on|at|via|through)\s+(.+)/gi,
|
|
47
|
+
/(?:desplegado|alojado|servido)\s+(?:en|vía|a través de)\s+(.+)/gi,
|
|
48
|
+
/(?:version|v)\s*(\d+\.\d+(?:\.\d+)?)/gi,
|
|
49
|
+
/(?:port|puerto)\s*(\d{2,5})/gi,
|
|
50
|
+
];
|
|
51
|
+
/**
|
|
52
|
+
* Extract structured content from a markdown file's text.
|
|
53
|
+
*/
|
|
54
|
+
function extractContent(text, filename) {
|
|
55
|
+
const lines = text.split('\n');
|
|
56
|
+
const topics = new Set();
|
|
57
|
+
const facts = [];
|
|
58
|
+
const decisions = [];
|
|
59
|
+
const technologies = new Set();
|
|
60
|
+
// Extract topics from headers
|
|
61
|
+
for (const line of lines) {
|
|
62
|
+
const headerMatch = line.match(/^#{1,3}\s+(.+)/);
|
|
63
|
+
if (headerMatch) {
|
|
64
|
+
const header = headerMatch[1].trim();
|
|
65
|
+
if (header.length > 3 && header.length < 100) {
|
|
66
|
+
topics.add(header);
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
// Extract from filename
|
|
71
|
+
const fileBaseName = filename.replace(/\.(md|txt)$/i, '').replace(/[_-]/g, ' ');
|
|
72
|
+
if (fileBaseName.length > 3) {
|
|
73
|
+
topics.add(fileBaseName);
|
|
74
|
+
}
|
|
75
|
+
const textLower = text.toLowerCase();
|
|
76
|
+
// Detect technologies
|
|
77
|
+
for (const tech of TECH_KEYWORDS) {
|
|
78
|
+
// Word boundary check to avoid partial matches
|
|
79
|
+
const regex = new RegExp(`\\b${tech.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}\\b`, 'i');
|
|
80
|
+
if (regex.test(textLower)) {
|
|
81
|
+
technologies.add(tech);
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
// Extract decisions
|
|
85
|
+
for (const pattern of DECISION_PATTERNS) {
|
|
86
|
+
pattern.lastIndex = 0;
|
|
87
|
+
let match;
|
|
88
|
+
while ((match = pattern.exec(text)) !== null) {
|
|
89
|
+
const decision = match[0].trim();
|
|
90
|
+
if (decision.length > 10 && decision.length < 200) {
|
|
91
|
+
decisions.push(cleanExtract(decision));
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
// Extract facts
|
|
96
|
+
for (const pattern of FACT_PATTERNS) {
|
|
97
|
+
pattern.lastIndex = 0;
|
|
98
|
+
let match;
|
|
99
|
+
while ((match = pattern.exec(text)) !== null) {
|
|
100
|
+
const fact = match[0].trim();
|
|
101
|
+
if (fact.length > 10 && fact.length < 200) {
|
|
102
|
+
facts.push(cleanExtract(fact));
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
// Generate summary from first substantial paragraph
|
|
107
|
+
const summary = generateSummary(lines);
|
|
108
|
+
return {
|
|
109
|
+
topics: [...topics].slice(0, 10),
|
|
110
|
+
facts: deduplicateStrings(facts).slice(0, 15),
|
|
111
|
+
decisions: deduplicateStrings(decisions).slice(0, 10),
|
|
112
|
+
technologies: [...technologies],
|
|
113
|
+
summary,
|
|
114
|
+
};
|
|
115
|
+
}
|
|
116
|
+
/**
|
|
117
|
+
* Clean an extracted string: remove markdown artifacts, trim whitespace.
|
|
118
|
+
*/
|
|
119
|
+
function cleanExtract(text) {
|
|
120
|
+
return text
|
|
121
|
+
.replace(/[`*_~\[\]]/g, '')
|
|
122
|
+
.replace(/\s+/g, ' ')
|
|
123
|
+
.trim();
|
|
124
|
+
}
|
|
125
|
+
/**
|
|
126
|
+
* Generate a one-line summary from the first meaningful paragraph.
|
|
127
|
+
*/
|
|
128
|
+
function generateSummary(lines) {
|
|
129
|
+
for (const line of lines) {
|
|
130
|
+
const trimmed = line.trim();
|
|
131
|
+
// Skip empty lines, headers, code blocks, frontmatter
|
|
132
|
+
if (!trimmed)
|
|
133
|
+
continue;
|
|
134
|
+
if (trimmed.startsWith('#'))
|
|
135
|
+
continue;
|
|
136
|
+
if (trimmed.startsWith('```'))
|
|
137
|
+
continue;
|
|
138
|
+
if (trimmed.startsWith('---'))
|
|
139
|
+
continue;
|
|
140
|
+
if (trimmed.startsWith('|'))
|
|
141
|
+
continue;
|
|
142
|
+
if (trimmed.startsWith('-') && trimmed.length < 20)
|
|
143
|
+
continue;
|
|
144
|
+
// Found a substantial line
|
|
145
|
+
if (trimmed.length > 30) {
|
|
146
|
+
const clean = cleanExtract(trimmed);
|
|
147
|
+
return clean.length > 150 ? clean.substring(0, 150) + '...' : clean;
|
|
148
|
+
}
|
|
149
|
+
}
|
|
150
|
+
return '';
|
|
151
|
+
}
|
|
152
|
+
/**
|
|
153
|
+
* Simple deduplication by normalized similarity.
|
|
154
|
+
*/
|
|
155
|
+
function deduplicateStrings(arr) {
|
|
156
|
+
const seen = new Set();
|
|
157
|
+
const result = [];
|
|
158
|
+
for (const item of arr) {
|
|
159
|
+
const normalized = item.toLowerCase().replace(/\s+/g, ' ').trim();
|
|
160
|
+
if (!seen.has(normalized)) {
|
|
161
|
+
seen.add(normalized);
|
|
162
|
+
result.push(item);
|
|
163
|
+
}
|
|
164
|
+
}
|
|
165
|
+
return result;
|
|
166
|
+
}
|
|
167
|
+
//# sourceMappingURL=extractor.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"extractor.js","sourceRoot":"","sources":["../../src/miner/extractor.ts"],"names":[],"mappings":";AAAA,iEAAiE;AACjE,+DAA+D;AAC/D,sDAAsD;;AA4DtD,wCAqEC;AAvHD,uCAAuC;AACvC,MAAM,aAAa,GAAG,IAAI,GAAG,CAAC;IAC5B,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,QAAQ,EAAE,SAAS,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM;IACxE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,SAAS,EAAE,SAAS,EAAE,KAAK;IAC5D,QAAQ,EAAE,QAAQ,EAAE,OAAO,EAAE,SAAS;IACtC,YAAY,EAAE,YAAY,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,OAAO;IAClF,UAAU,EAAE,UAAU,EAAE,SAAS,EAAE,YAAY,EAAE,OAAO,EAAE,OAAO,EAAE,QAAQ;IAC3E,QAAQ,EAAE,YAAY,EAAE,WAAW,EAAE,SAAS;IAC9C,KAAK,EAAE,KAAK,EAAE,OAAO,EAAE,YAAY,EAAE,QAAQ,EAAE,SAAS;IACxD,QAAQ,EAAE,QAAQ,EAAE,WAAW;IAC/B,WAAW,EAAE,SAAS,EAAE,QAAQ,EAAE,QAAQ;IAC1C,UAAU,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM;IACjC,SAAS,EAAE,MAAM,EAAE,MAAM,EAAE,WAAW;IACtC,QAAQ,EAAE,QAAQ,EAAE,QAAQ,EAAE,QAAQ,EAAE,QAAQ,EAAE,QAAQ;IAC1D,UAAU,EAAE,OAAO,EAAE,SAAS,EAAE,cAAc;IAC9C,WAAW,EAAE,iBAAiB,EAAE,QAAQ,EAAE,KAAK,EAAE,IAAI;IACrD,WAAW,EAAE,mBAAmB,EAAE,eAAe;IACjD,KAAK,EAAE,WAAW,EAAE,OAAO,EAAE,eAAe;IAC5C,KAAK,EAAE,wBAAwB;CAChC,CAAC,CAAC;AAEH,6CAA6C;AAC7C,MAAM,iBAAiB,GAAG;IACxB,UAAU;IACV,0HAA0H;IAC1H,yHAAyH;IACzH,UAAU;IACV,yEAAyE;IACzE,+FAA+F;IAC/F,SAAS;IACT,uEAAuE;IACvE,SAAS;IACT,8DAA8D;IAC9D,UAAU;IACV,uEAAuE;CACxE,CAAC;AAEF,0BAA0B;AAC1B,MAAM,aAAa,GAAG;IACpB,mFAAmF;IACnF,0EAA0E;IAC1E,gEAAgE;IAChE,kEAAkE;IAClE,wCAAwC;IACxC,+BAA+B;CAChC,CAAC;AAEF;;GAEG;AACH,SAAgB,cAAc,CAAC,IAAY,EAAE,QAAgB;IAC3D,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAC/B,MAAM,MAAM,GAAgB,IAAI,GAAG,EAAE,CAAC;IACtC,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,MAAM,SAAS,GAAa,EAAE,CAAC;IAC/B,MAAM,YAAY,GAAgB,IAAI,GAAG,EAAE,CAAC;IAE5C,8BAA8B;IAC9B,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,WAAW,GAAG,IAAI,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAC;QACjD,IAAI,WAAW,EAAE,CAAC;YAChB,MAAM,MAAM,GAAG,WAAW,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;YACrC,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,IAAI,MAAM,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC;gBAC7C,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;YACrB,CAAC;QACH,CAAC;IACH,CAAC;IAED,wBAAwB;IACxB,MAAM,YAAY,GAAG,QAAQ,CAAC,OAAO,CAAC,cAAc,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;IAChF,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC5B,MAAM,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC;IAC3B,CAAC;IAED,MAAM,SAAS,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC;IAErC,sBAAsB;IACtB,KAAK,MAAM,IAAI,IAAI,aAAa,EAAE,CAAC;QACjC,+CAA+C;QAC/C,MAAM,KAAK,GAAG,IAAI,MAAM,CAAC,MAAM,IAAI,CAAC,OAAO,CAAC,qBAAqB,EAAE,MAAM,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;QACtF,IAAI,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC;YAC1B,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACzB,CAAC;IACH,CAAC;IAED,oBAAoB;IACpB,KAAK,MAAM,OAAO,IAAI,iBAAiB,EAAE,CAAC;QACxC,OAAO,CAAC,SAAS,GAAG,CAAC,CAAC;QACtB,IAAI,KAAK,CAAC;QACV,OAAO,CAAC,KAAK,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;YAC7C,MAAM,QAAQ,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;YACjC,IAAI,QAAQ,CAAC,MAAM,GAAG,EAAE,IAAI,QAAQ,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC;gBAClD,SAAS,CAAC,IAAI,CAAC,YAAY,CAAC,QAAQ,CAAC,CAAC,CAAC;YACzC,CAAC;QACH,CAAC;IACH,CAAC;IAED,gBAAgB;IAChB,KAAK,MAAM,OAAO,IAAI,aAAa,EAAE,CAAC;QACpC,OAAO,CAAC,SAAS,GAAG,CAAC,CAAC;QACtB,IAAI,KAAK,CAAC;QACV,OAAO,CAAC,KAAK,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;YAC7C,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;YAC7B,IAAI,IAAI,CAAC,MAAM,GAAG,EAAE,IAAI,IAAI,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC;gBAC1C,KAAK,CAAC,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC;YACjC,CAAC;QACH,CAAC;IACH,CAAC;IAED,oDAAoD;IACpD,MAAM,OAAO,GAAG,eAAe,CAAC,KAAK,CAAC,CAAC;IAEvC,OAAO;QACL,MAAM,EAAE,CAAC,GAAG,MAAM,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC;QAChC,KAAK,EAAE,kBAAkB,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC;QAC7C,SAAS,EAAE,kBAAkB,CAAC,SAAS,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC;QACrD,YAAY,EAAE,CAAC,GAAG,YAAY,CAAC;QAC/B,OAAO;KACR,CAAC;AACJ,CAAC;AAED;;GAEG;AACH,SAAS,YAAY,CAAC,IAAY;IAChC,OAAO,IAAI;SACR,OAAO,CAAC,aAAa,EAAE,EAAE,CAAC;SAC1B,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC;SACpB,IAAI,EAAE,CAAC;AACZ,CAAC;AAED;;GAEG;AACH,SAAS,eAAe,CAAC,KAAe;IACtC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;QAC5B,sDAAsD;QACtD,IAAI,CAAC,OAAO;YAAE,SAAS;QACvB,IAAI,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC;YAAE,SAAS;QACtC,IAAI,OAAO,CAAC,UAAU,CAAC,KAAK,CAAC;YAAE,SAAS;QACxC,IAAI,OAAO,CAAC,UAAU,CAAC,KAAK,CAAC;YAAE,SAAS;QACxC,IAAI,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC;YAAE,SAAS;QACtC,IAAI,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,OAAO,CAAC,MAAM,GAAG,EAAE;YAAE,SAAS;QAE7D,2BAA2B;QAC3B,IAAI,OAAO,CAAC,MAAM,GAAG,EAAE,EAAE,CAAC;YACxB,MAAM,KAAK,GAAG,YAAY,CAAC,OAAO,CAAC,CAAC;YACpC,OAAO,KAAK,CAAC,MAAM,GAAG,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,EAAE,GAAG,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC;QACtE,CAAC;IACH,CAAC;IACD,OAAO,EAAE,CAAC;AACZ,CAAC;AAED;;GAEG;AACH,SAAS,kBAAkB,CAAC,GAAa;IACvC,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/B,MAAM,MAAM,GAAa,EAAE,CAAC;IAC5B,KAAK,MAAM,IAAI,IAAI,GAAG,EAAE,CAAC;QACvB,MAAM,UAAU,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;QAClE,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,EAAE,CAAC;YAC1B,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;YACrB,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACpB,CAAC;IACH,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC"}
|