brainfood 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +180 -0
- package/dist/crawl.d.ts +7 -0
- package/dist/crawl.js +380 -0
- package/dist/crawl.js.map +1 -0
- package/dist/extract.d.ts +2 -0
- package/dist/extract.js +250 -0
- package/dist/extract.js.map +1 -0
- package/dist/index.d.ts +2 -0
- package/dist/index.js +199 -0
- package/dist/index.js.map +1 -0
- package/dist/local.d.ts +2 -0
- package/dist/local.js +110 -0
- package/dist/local.js.map +1 -0
- package/dist/output.d.ts +2 -0
- package/dist/output.js +235 -0
- package/dist/output.js.map +1 -0
- package/dist/sitemap.d.ts +6 -0
- package/dist/sitemap.js +71 -0
- package/dist/sitemap.js.map +1 -0
- package/dist/structure.d.ts +3 -0
- package/dist/structure.js +423 -0
- package/dist/structure.js.map +1 -0
- package/dist/summarize.d.ts +2 -0
- package/dist/summarize.js +57 -0
- package/dist/summarize.js.map +1 -0
- package/dist/types.d.ts +131 -0
- package/dist/types.js +2 -0
- package/dist/types.js.map +1 -0
- package/dist/utils.d.ts +14 -0
- package/dist/utils.js +151 -0
- package/dist/utils.js.map +1 -0
- package/package.json +69 -0
package/dist/utils.d.ts
ADDED
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
export declare const DEFAULT_USER_AGENT = "brainfood/1.0 (+https://github.com/Capxel/brainfood)";
|
|
2
|
+
export declare function sleep(ms: number): Promise<void>;
|
|
3
|
+
export declare function normalizeWhitespace(text: string): string;
|
|
4
|
+
export declare function ensureDirectory(targetDir: string): Promise<void>;
|
|
5
|
+
export declare function sanitizeSegment(value: string): string;
|
|
6
|
+
export declare function slugify(value: string): string;
|
|
7
|
+
export declare function normalizeUrl(rawUrl: string): string;
|
|
8
|
+
export declare function urlToRelativeOutputPath(rawUrl: string): string;
|
|
9
|
+
export declare function localPathToRelativeOutputPath(rootDir: string, filePath: string): string;
|
|
10
|
+
export declare function shortHash(input: string): string;
|
|
11
|
+
export declare function fallbackSummary(content: string): string | null;
|
|
12
|
+
export declare function sentenceSummary(content: string): string | null;
|
|
13
|
+
export declare function safeIsoDate(value: string | null | undefined): string | null;
|
|
14
|
+
export declare function packageVersion(): string;
|
package/dist/utils.js
ADDED
|
@@ -0,0 +1,151 @@
|
|
|
1
|
+
import { createHash } from 'node:crypto';
|
|
2
|
+
import { mkdir } from 'node:fs/promises';
|
|
3
|
+
import path from 'node:path';
|
|
4
|
+
export const DEFAULT_USER_AGENT = 'brainfood/1.0 (+https://github.com/Capxel/brainfood)';
|
|
5
|
+
export function sleep(ms) {
|
|
6
|
+
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
7
|
+
}
|
|
8
|
+
export function normalizeWhitespace(text) {
|
|
9
|
+
return text
|
|
10
|
+
.replace(/\r/g, '\n')
|
|
11
|
+
.replace(/\u00a0/g, ' ')
|
|
12
|
+
.replace(/[ \t]+\n/g, '\n')
|
|
13
|
+
.replace(/\n{3,}/g, '\n\n')
|
|
14
|
+
.replace(/[ \t]{2,}/g, ' ')
|
|
15
|
+
.trim();
|
|
16
|
+
}
|
|
17
|
+
export async function ensureDirectory(targetDir) {
|
|
18
|
+
await mkdir(targetDir, { recursive: true });
|
|
19
|
+
}
|
|
20
|
+
export function sanitizeSegment(value) {
|
|
21
|
+
const cleaned = value
|
|
22
|
+
.replace(/%[0-9A-F]{2}/gi, '-')
|
|
23
|
+
.replace(/[^a-zA-Z0-9._-]+/g, '-')
|
|
24
|
+
.replace(/^-+|-+$/g, '')
|
|
25
|
+
.toLowerCase();
|
|
26
|
+
return cleaned || 'index';
|
|
27
|
+
}
|
|
28
|
+
export function slugify(value) {
|
|
29
|
+
const normalized = value.normalize('NFKD').replace(/[\u0300-\u036f]/g, '');
|
|
30
|
+
const slug = normalized
|
|
31
|
+
.replace(/[^a-zA-Z0-9]+/g, '-')
|
|
32
|
+
.replace(/^-+|-+$/g, '')
|
|
33
|
+
.toLowerCase();
|
|
34
|
+
return slug || 'untitled';
|
|
35
|
+
}
|
|
36
|
+
export function normalizeUrl(rawUrl) {
|
|
37
|
+
const url = new URL(rawUrl);
|
|
38
|
+
url.hash = '';
|
|
39
|
+
if ((url.protocol === 'https:' && url.port === '443') || (url.protocol === 'http:' && url.port === '80')) {
|
|
40
|
+
url.port = '';
|
|
41
|
+
}
|
|
42
|
+
if (url.pathname !== '/' && url.pathname.endsWith('/')) {
|
|
43
|
+
url.pathname = url.pathname.slice(0, -1);
|
|
44
|
+
}
|
|
45
|
+
return url.toString();
|
|
46
|
+
}
|
|
47
|
+
export function urlToRelativeOutputPath(rawUrl) {
|
|
48
|
+
const url = new URL(rawUrl);
|
|
49
|
+
const host = sanitizeSegment(url.hostname);
|
|
50
|
+
const segments = url.pathname
|
|
51
|
+
.split('/')
|
|
52
|
+
.filter(Boolean)
|
|
53
|
+
.map((segment) => sanitizeSegment(segment));
|
|
54
|
+
if (!segments.length) {
|
|
55
|
+
return path.posix.join(host, 'index');
|
|
56
|
+
}
|
|
57
|
+
const lastSegment = segments[segments.length - 1] ?? 'index';
|
|
58
|
+
if (lastSegment.includes('.')) {
|
|
59
|
+
const withoutExtension = lastSegment.replace(/\.[a-z0-9]+$/i, '') || 'index';
|
|
60
|
+
segments[segments.length - 1] = withoutExtension;
|
|
61
|
+
}
|
|
62
|
+
return path.posix.join(host, ...segments);
|
|
63
|
+
}
|
|
64
|
+
export function localPathToRelativeOutputPath(rootDir, filePath) {
|
|
65
|
+
const relativePath = path.relative(rootDir, filePath);
|
|
66
|
+
const segments = relativePath
|
|
67
|
+
.split(path.sep)
|
|
68
|
+
.filter(Boolean)
|
|
69
|
+
.map((segment) => sanitizeSegment(segment.replace(/\.[^.]+$/, '')));
|
|
70
|
+
return segments.length ? path.posix.join(...segments) : 'index';
|
|
71
|
+
}
|
|
72
|
+
export function shortHash(input) {
|
|
73
|
+
return createHash('sha256').update(input).digest('hex').slice(0, 12);
|
|
74
|
+
}
|
|
75
|
+
const SUMMARY_STOPWORDS = new Set([
|
|
76
|
+
'a', 'an', 'and', 'are', 'as', 'at', 'be', 'been', 'being', 'but', 'by', 'for', 'from', 'had', 'has', 'have',
|
|
77
|
+
'he', 'her', 'his', 'in', 'into', 'is', 'it', 'its', 'of', 'on', 'or', 'our', 'she', 'that', 'the', 'their',
|
|
78
|
+
'them', 'they', 'this', 'to', 'was', 'we', 'were', 'will', 'with', 'you', 'your'
|
|
79
|
+
]);
|
|
80
|
+
function summarizeText(sentences) {
|
|
81
|
+
return normalizeWhitespace(sentences.join(' ')) || null;
|
|
82
|
+
}
|
|
83
|
+
export function fallbackSummary(content) {
|
|
84
|
+
const normalized = normalizeWhitespace(content);
|
|
85
|
+
if (!normalized) {
|
|
86
|
+
return null;
|
|
87
|
+
}
|
|
88
|
+
const sentences = normalized
|
|
89
|
+
.split(/(?<=[.!?])\s+/)
|
|
90
|
+
.map((sentence) => sentence.trim())
|
|
91
|
+
.filter(Boolean);
|
|
92
|
+
return normalizeWhitespace(sentences.slice(0, 2).join(' ')).slice(0, 320) || null;
|
|
93
|
+
}
|
|
94
|
+
export function sentenceSummary(content) {
|
|
95
|
+
const normalized = normalizeWhitespace(content);
|
|
96
|
+
if (!normalized) {
|
|
97
|
+
return null;
|
|
98
|
+
}
|
|
99
|
+
const sentences = normalized
|
|
100
|
+
.split(/(?<=[.!?])\s+/)
|
|
101
|
+
.map((sentence) => sentence.trim())
|
|
102
|
+
.filter((sentence) => sentence.length >= 20);
|
|
103
|
+
if (sentences.length === 0) {
|
|
104
|
+
return fallbackSummary(content);
|
|
105
|
+
}
|
|
106
|
+
const scored = sentences.map((sentence, index) => {
|
|
107
|
+
const capitalizedWords = sentence.match(/\b[A-Z][a-z]+(?:[A-Z][A-Za-z]+)?\b/g)?.length || 0;
|
|
108
|
+
const numbers = sentence.match(/\b\d+(?:[.,]\d+)?\b/g)?.length || 0;
|
|
109
|
+
const uniqueTerms = new Set((sentence.toLowerCase().match(/\b[a-z][a-z0-9-]{2,}\b/g) || []).filter((term) => !SUMMARY_STOPWORDS.has(term))).size;
|
|
110
|
+
return {
|
|
111
|
+
sentence,
|
|
112
|
+
index,
|
|
113
|
+
score: capitalizedWords * 3 + numbers * 2 + uniqueTerms
|
|
114
|
+
};
|
|
115
|
+
});
|
|
116
|
+
const selected = scored
|
|
117
|
+
.filter((entry) => entry.score > 0)
|
|
118
|
+
.sort((left, right) => right.score - left.score || left.index - right.index)
|
|
119
|
+
.slice(0, 3)
|
|
120
|
+
.sort((left, right) => left.index - right.index)
|
|
121
|
+
.map((entry) => entry.sentence);
|
|
122
|
+
if (selected.length === 0) {
|
|
123
|
+
return fallbackSummary(content);
|
|
124
|
+
}
|
|
125
|
+
const kept = [];
|
|
126
|
+
for (const sentence of selected) {
|
|
127
|
+
const candidate = summarizeText([...kept, sentence]);
|
|
128
|
+
if (!candidate) {
|
|
129
|
+
continue;
|
|
130
|
+
}
|
|
131
|
+
if (candidate.length > 500) {
|
|
132
|
+
if (kept.length === 0) {
|
|
133
|
+
return candidate;
|
|
134
|
+
}
|
|
135
|
+
break;
|
|
136
|
+
}
|
|
137
|
+
kept.push(sentence);
|
|
138
|
+
}
|
|
139
|
+
return summarizeText(kept)?.slice(0, 500) || fallbackSummary(content);
|
|
140
|
+
}
|
|
141
|
+
export function safeIsoDate(value) {
|
|
142
|
+
if (!value) {
|
|
143
|
+
return null;
|
|
144
|
+
}
|
|
145
|
+
const timestamp = Date.parse(value);
|
|
146
|
+
return Number.isNaN(timestamp) ? null : new Date(timestamp).toISOString();
|
|
147
|
+
}
|
|
148
|
+
export function packageVersion() {
|
|
149
|
+
return process.env.npm_package_version || '1.0.0';
|
|
150
|
+
}
|
|
151
|
+
//# sourceMappingURL=utils.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"utils.js","sourceRoot":"","sources":["../src/utils.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AACzC,OAAO,IAAI,MAAM,WAAW,CAAC;AAE7B,MAAM,CAAC,MAAM,kBAAkB,GAAG,sDAAsD,CAAC;AAEzF,MAAM,UAAU,KAAK,CAAC,EAAU;IAC9B,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;AAC3D,CAAC;AAED,MAAM,UAAU,mBAAmB,CAAC,IAAY;IAC9C,OAAO,IAAI;SACR,OAAO,CAAC,KAAK,EAAE,IAAI,CAAC;SACpB,OAAO,CAAC,SAAS,EAAE,GAAG,CAAC;SACvB,OAAO,CAAC,WAAW,EAAE,IAAI,CAAC;SAC1B,OAAO,CAAC,SAAS,EAAE,MAAM,CAAC;SAC1B,OAAO,CAAC,YAAY,EAAE,GAAG,CAAC;SAC1B,IAAI,EAAE,CAAC;AACZ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,eAAe,CAAC,SAAiB;IACrD,MAAM,KAAK,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;AAC9C,CAAC;AAED,MAAM,UAAU,eAAe,CAAC,KAAa;IAC3C,MAAM,OAAO,GAAG,KAAK;SAClB,OAAO,CAAC,gBAAgB,EAAE,GAAG,CAAC;SAC9B,OAAO,CAAC,mBAAmB,EAAE,GAAG,CAAC;SACjC,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC;SACvB,WAAW,EAAE,CAAC;IAEjB,OAAO,OAAO,IAAI,OAAO,CAAC;AAC5B,CAAC;AAED,MAAM,UAAU,OAAO,CAAC,KAAa;IACnC,MAAM,UAAU,GAAG,KAAK,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,kBAAkB,EAAE,EAAE,CAAC,CAAC;IAC3E,MAAM,IAAI,GAAG,UAAU;SACpB,OAAO,CAAC,gBAAgB,EAAE,GAAG,CAAC;SAC9B,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC;SACvB,WAAW,EAAE,CAAC;IAEjB,OAAO,IAAI,IAAI,UAAU,CAAC;AAC5B,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,MAAc;IACzC,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,CAAC;IAC5B,GAAG,CAAC,IAAI,GAAG,EAAE,CAAC;IAEd,IAAI,CAAC,GAAG,CAAC,QAAQ,KAAK,QAAQ,IAAI,GAAG,CAAC,IAAI,KAAK,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,KAAK,OAAO,IAAI,GAAG,CAAC,IAAI,KAAK,IAAI,CAAC,EAAE,CAAC;QACzG,GAAG,CAAC,IAAI,GAAG,EAAE,CAAC;IAChB,CAAC;IAED,IAAI,GAAG,CAAC,QAAQ,KAAK,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;QACvD,GAAG,CAAC,QAAQ,GAAG,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IAC3C,CAAC;IAED,OAAO,GAAG,CAAC,QAAQ,EAAE,CAAC;AACxB,CAAC;AAED,MAAM,UAAU,uBAAuB,CAAC,MAAc;IACpD,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,CAAC;IAC5B,MAAM,IAAI,GAAG,eAAe,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;IAC3C,MAAM,QAAQ,GAAG,GAAG,CAAC,QAAQ;SAC1B,KAAK,CAAC,GAAG,CAAC;SACV,MAAM,CAAC,OAAO,CAAC;SACf,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,eAAe,CAAC,OAAO,CAAC,CAAC,CAAC;IAE9C,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC;QACrB,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IACxC,CAAC;IAED,MAAM,WAAW,GAAG,QAAQ,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,IAAI,OAAO,CAAC;IAC7D,IAAI,WAAW,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;QAC9B,MAAM,gBAAgB,GAAG,WAAW,CAAC,OAAO,CAAC,eAAe,EAAE,EAAE,CAAC,IAAI,OAAO,CAAC;QAC7E,QAAQ,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,GAAG,gBAAgB,CAAC;IACnD,CAAC;IAED,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,EAAE,GAAG,QAAQ,CAAC,CAAC;AAC5C,CAAC;AAED,MAAM,UAAU,6BAA6B,CAAC,OAAe,EAAE,QAAgB;IAC7E,MAAM,YAAY,GAAG,IAAI,CAAC,QAAQ,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;IACtD,MAAM,QAAQ,GAAG,YAAY;SAC1B,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC;SACf,MAAM,CAAC,OAAO,CAAC;SACf,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,eAAe,CAAC,OAAO,CAAC,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC;IAEtE,OAAO,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC;AAClE,CAAC;AAED,MAAM,UAAU,SAAS,CAAC,KAAa;IACrC,OAAO,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;AACvE,CAAC;AAED,MAAM,iBAAiB,GAAG,IAAI,GAAG,CAAC;IAChC,GAAG,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM;IAC5G,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO;IAC3G,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM;CACjF,CAAC,CAAC;AAEH,SAAS,aAAa,CAAC,SAAmB;IACxC,OAAO,mBAAmB,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,IAAI,CAAC;AAC1D,CAAC;AAED,MAAM,UAAU,eAAe,CAAC,OAAe;IAC7C,MAAM,UAAU,GAAG,mBAAmB,CAAC,OAAO,CAAC,CAAC;IAChD,IAAI,CAAC,UAAU,EAAE,CAAC;QAChB,OAAO,IAAI,CAAC;IACd,CAAC;IAED,MAAM,SAAS,GAAG,UAAU;SACzB,KAAK,CAAC,eAAe,CAAC;SACtB,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC;SAClC,MAAM,CAAC,OAAO,CAAC,CAAC;IAEnB,OAAO,mBAAmB,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,IAAI,IAAI,CAAC;AACpF,CAAC;AAED,MAAM,UAAU,eAAe,CAAC,OAAe;IAC7C,MAAM,UAAU,GAAG,mBAAmB,CAAC,OAAO,CAAC,CAAC;IAChD,IAAI,CAAC,UAAU,EAAE,CAAC;QAChB,OAAO,IAAI,CAAC;IACd,CAAC;IAED,MAAM,SAAS,GAAG,UAAU;SACzB,KAAK,CAAC,eAAe,CAAC;SACtB,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC;SAClC,MAAM,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC;IAE/C,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC3B,OAAO,eAAe,CAAC,OAAO,CAAC,CAAC;IAClC,CAAC;IAED,MAAM,MAAM,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,KAAK,EAAE,EAAE;QAC/C,MAAM,gBAAgB,GAAG,QAAQ,CAAC,KAAK,CAAC,qCAAqC,CAAC,EAAE,MAAM,IAAI,CAAC,CAAC;QAC5F,MAAM,OAAO,GAAG,QAAQ,CAAC,KAAK,CAAC,sBAAsB,CAAC,EAAE,MAAM,IAAI,CAAC,CAAC;QACpE,MAAM,WAAW,GAAG,IAAI,GAAG,CACzB,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,yBAAyB,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,iBAAiB,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAC/G,CAAC,IAAI,CAAC;QAEP,OAAO;YACL,QAAQ;YACR,KAAK;YACL,KAAK,EAAE,gBAAgB,GAAG,CAAC,GAAG,OAAO,GAAG,CAAC,GAAG,WAAW;SACxD,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,MAAM,QAAQ,GAAG,MAAM;SACpB,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,KAAK,GAAG,CAAC,CAAC;SAClC,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC;SAC3E,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC;SACX,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC;SAC/C,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;IAElC,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC1B,OAAO,eAAe,CAAC,OAAO,CAAC,CAAC;IAClC,CAAC;IAED,MAAM,IAAI,GAAa,EAAE,CAAC;IAC1B,KAAK,MAAM,QAAQ,IAAI,QAAQ,EAAE,CAAC;QAChC,MAAM,SAAS,GAAG,aAAa,CAAC,CAAC,GAAG,IAAI,EAAE,QAAQ,CAAC,CAAC,CAAC;QACrD,IAAI,CAAC,SAAS,EAAE,CAAC;YACf,SAAS;QACX,CAAC;QAED,IAAI,SAAS,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC;YAC3B,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACtB,OAAO,SAAS,CAAC;YACnB,CAAC;YACD,MAAM;QACR,CAAC;QAED,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IACtB,CAAC;IAED,OAAO,aAAa,CAAC,IAAI,CAAC,EAAE,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,IAAI,eAAe,CAAC,OAAO,CAAC,CAAC;AACxE,CAAC;AAED,MAAM,UAAU,WAAW,CAAC,KAAgC;IAC1D,IAAI,CAAC,KAAK,EAAE,CAAC;QACX,OAAO,IAAI,CAAC;IACd,CAAC;IAED,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IACpC,OAAO,MAAM,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,SAAS,CAAC,CAAC,WAAW,EAAE,CAAC;AAC5E,CAAC;AAED,MAAM,UAAU,cAAc;IAC5B,OAAO,OAAO,CAAC,GAAG,CAAC,mBAAmB,IAAI,OAAO,CAAC;AACpD,CAAC"}
|
package/package.json
ADDED
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "brainfood",
|
|
3
|
+
"version": "1.0.0",
|
|
4
|
+
"description": "Structured knowledge for hungry agents.",
|
|
5
|
+
"type": "module",
|
|
6
|
+
"license": "MIT",
|
|
7
|
+
"author": "Capxel",
|
|
8
|
+
"homepage": "https://github.com/Capxel/brainfood#readme",
|
|
9
|
+
"repository": {
|
|
10
|
+
"type": "git",
|
|
11
|
+
"url": "git+https://github.com/Capxel/brainfood.git"
|
|
12
|
+
},
|
|
13
|
+
"bugs": {
|
|
14
|
+
"url": "https://github.com/Capxel/brainfood/issues"
|
|
15
|
+
},
|
|
16
|
+
"keywords": [
|
|
17
|
+
"cli",
|
|
18
|
+
"crawler",
|
|
19
|
+
"knowledge-graph",
|
|
20
|
+
"llm",
|
|
21
|
+
"markdown",
|
|
22
|
+
"sitemap",
|
|
23
|
+
"typescript",
|
|
24
|
+
"brainfood",
|
|
25
|
+
"obsidian",
|
|
26
|
+
"ai-agent",
|
|
27
|
+
"knowledge",
|
|
28
|
+
"structured-data",
|
|
29
|
+
"pdf",
|
|
30
|
+
"youtube",
|
|
31
|
+
"transcript",
|
|
32
|
+
"openclaw"
|
|
33
|
+
],
|
|
34
|
+
"engines": {
|
|
35
|
+
"node": ">=18"
|
|
36
|
+
},
|
|
37
|
+
"bin": {
|
|
38
|
+
"brainfood": "dist/index.js"
|
|
39
|
+
},
|
|
40
|
+
"files": [
|
|
41
|
+
"dist",
|
|
42
|
+
"README.md",
|
|
43
|
+
"LICENSE"
|
|
44
|
+
],
|
|
45
|
+
"scripts": {
|
|
46
|
+
"build": "tsc -p tsconfig.json",
|
|
47
|
+
"check": "tsc --noEmit -p tsconfig.json",
|
|
48
|
+
"dev": "tsx src/index.ts",
|
|
49
|
+
"crawl": "tsx src/index.ts crawl",
|
|
50
|
+
"local": "tsx src/index.ts local",
|
|
51
|
+
"sitemap": "tsx src/index.ts sitemap",
|
|
52
|
+
"prepare": "npm run build"
|
|
53
|
+
},
|
|
54
|
+
"dependencies": {
|
|
55
|
+
"@mozilla/readability": "^0.6.0",
|
|
56
|
+
"cheerio": "^1.1.2",
|
|
57
|
+
"commander": "^14.0.1",
|
|
58
|
+
"jsdom": "^26.1.0",
|
|
59
|
+
"mammoth": "^1.12.0",
|
|
60
|
+
"node-fetch": "^3.3.2",
|
|
61
|
+
"pdf-parse": "^2.4.5"
|
|
62
|
+
},
|
|
63
|
+
"devDependencies": {
|
|
64
|
+
"@types/jsdom": "^21.1.7",
|
|
65
|
+
"@types/node": "^24.5.2",
|
|
66
|
+
"tsx": "^4.20.5",
|
|
67
|
+
"typescript": "^5.9.2"
|
|
68
|
+
}
|
|
69
|
+
}
|