@franyerhiguerey/ai-chat-ingest 0.0.0-stage → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +48 -2
- package/dist/index.d.ts +86 -0
- package/dist/index.js +312 -0
- package/dist/index.js.map +1 -0
- package/package.json +66 -3
package/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Franyer Higuerey
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
package/README.md
CHANGED
|
@@ -1,3 +1,49 @@
|
|
|
1
|
-
#
|
|
1
|
+
# @franyerhiguerey/ai-chat-ingest
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
Turns files a site owner uploads into ai-chat knowledge documents and catalogs. Node only. Use it in your dashboard's upload handler, then hand the documents to the chat through a source (see [docs/integration.md](../../docs/integration.md)).
|
|
4
|
+
|
|
5
|
+
```sh
|
|
6
|
+
pnpm add @franyerhiguerey/ai-chat-ingest
|
|
7
|
+
# Only for the formats you accept:
|
|
8
|
+
pnpm add unpdf # PDF
|
|
9
|
+
pnpm add mammoth # DOCX
|
|
10
|
+
```
|
|
11
|
+
|
|
12
|
+
```ts
|
|
13
|
+
import { IngestError, toKnowledgeDoc } from '@franyerhiguerey/ai-chat-ingest';
|
|
14
|
+
|
|
15
|
+
try {
|
|
16
|
+
const doc = await toKnowledgeDoc({ name: file.name, type: file.type, bytes });
|
|
17
|
+
await db.saveKnowledgeDoc(doc); // { file: 'lista-de-precios.md', content: '…' }
|
|
18
|
+
} catch (error) {
|
|
19
|
+
if (error instanceof IngestError)
|
|
20
|
+
showToOwner(error.message); // error.code for logic
|
|
21
|
+
else throw error;
|
|
22
|
+
}
|
|
23
|
+
```
|
|
24
|
+
|
|
25
|
+
| Input | Output |
|
|
26
|
+
| --------------- | ------------------------------------ |
|
|
27
|
+
| `.md`, `.txt` | The text, line endings normalized |
|
|
28
|
+
| `.html`, `.htm` | Text without scripts, styles or tags |
|
|
29
|
+
| `.pdf` | The text layer, via `unpdf` |
|
|
30
|
+
| `.docx` | The document text, via `mammoth` |
|
|
31
|
+
|
|
32
|
+
The type comes from the file extension, then from `type` (MIME).
|
|
33
|
+
|
|
34
|
+
**Errors** (`IngestError.code`): `unsupported_type`, `too_large` (over `maxBytes`, default 10 MB), `empty` (no text, e.g. a scanned PDF) and `missing_parser` (install `unpdf` or `mammoth`).
|
|
35
|
+
|
|
36
|
+
**File names.** `doc.file` is a lowercase slug ending in `.md` (`safeFileName()`), so it is safe as a file name or key: `../../etc/passwd` becomes `etc-passwd.md`.
|
|
37
|
+
|
|
38
|
+
Convert at upload time, not per chat: parsing is slow, and stored text keeps the bot's prompt fast and identical between requests, which prompt caching needs.
|
|
39
|
+
|
|
40
|
+
## Catalogs
|
|
41
|
+
|
|
42
|
+
```ts
|
|
43
|
+
import { parseCatalog } from '@franyerhiguerey/ai-chat-ingest';
|
|
44
|
+
|
|
45
|
+
const { items, errors } = parseCatalog(await file.text()); // CSV or JSON
|
|
46
|
+
// errors: [{ row: 4, message: 'price "abc" is not a number' }], the other rows still import
|
|
47
|
+
```
|
|
48
|
+
|
|
49
|
+
CSV follows RFC 4180 (quotes, newlines in quotes), with `,` or `;` detected from the header. Column names are matched in Spanish and English (`nombre`/`name`, `precio`/`price`, `enlace`/`url`…); unknown columns become searchable `attributes`. Prices like `0,25` or `1.234,50` are read as numbers. Links must be http(s). A duplicated id keeps its first row. See [docs/integration.md](../../docs/integration.md#4-catalog) for the full column list.
|
package/dist/index.d.ts
ADDED
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
/** The same shape as ai-chat-core's `KnowledgeDoc`, so the output can be handed to a source. */
|
|
2
|
+
interface KnowledgeDoc {
|
|
3
|
+
file: string;
|
|
4
|
+
content: string;
|
|
5
|
+
}
|
|
6
|
+
interface UploadedFile {
|
|
7
|
+
/** The original file name; its extension picks the parser. */
|
|
8
|
+
name: string;
|
|
9
|
+
/** MIME type, used when the name has no known extension. */
|
|
10
|
+
type?: string;
|
|
11
|
+
bytes: Uint8Array;
|
|
12
|
+
}
|
|
13
|
+
interface IngestOptions {
|
|
14
|
+
/** Default 10 MB. */
|
|
15
|
+
maxBytes?: number;
|
|
16
|
+
}
|
|
17
|
+
/**
|
|
18
|
+
* A file name that is safe to write anywhere: lowercase ASCII words joined by
|
|
19
|
+
* dashes, always ending in `.md`. Path separators, dots and accents can't
|
|
20
|
+
* survive it, so `../../etc/passwd` becomes `etc-passwd.md`.
|
|
21
|
+
*/
|
|
22
|
+
declare function safeFileName(name: string): string;
|
|
23
|
+
/**
|
|
24
|
+
* Converts an uploaded file to a knowledge document. Run it when the owner
|
|
25
|
+
* uploads, not during a chat: PDF and DOCX parsing is slow, and storing the
|
|
26
|
+
* text keeps the bot's prompt fast and identical between requests.
|
|
27
|
+
*/
|
|
28
|
+
declare function toKnowledgeDoc(file: UploadedFile, options?: IngestOptions): Promise<KnowledgeDoc>;
|
|
29
|
+
|
|
30
|
+
/**
|
|
31
|
+
* Plain text from an HTML page: scripts, styles and comments dropped, block
|
|
32
|
+
* elements on their own lines. Not a sanitizer: the output is only ever text.
|
|
33
|
+
*/
|
|
34
|
+
declare function htmlToText(html: string): string;
|
|
35
|
+
|
|
36
|
+
type IngestErrorCode = 'unsupported_type' | 'too_large' | 'empty' | 'missing_parser';
|
|
37
|
+
/** Why a file couldn't become knowledge. `message` is meant for the site owner, in English. */
|
|
38
|
+
declare class IngestError extends Error {
|
|
39
|
+
readonly code: IngestErrorCode;
|
|
40
|
+
constructor(code: IngestErrorCode, message: string, options?: {
|
|
41
|
+
cause?: unknown;
|
|
42
|
+
});
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
/**
|
|
46
|
+
* Catalog import: a CSV or JSON file from the site owner becomes catalog items
|
|
47
|
+
* in the shape ai-chat-core expects. Bad rows are reported, the rest import.
|
|
48
|
+
*/
|
|
49
|
+
/** The same shape as ai-chat-core's `CatalogItemInput`; core validates it again on load. */
|
|
50
|
+
interface CatalogItem {
|
|
51
|
+
id: string;
|
|
52
|
+
name: string;
|
|
53
|
+
description?: string;
|
|
54
|
+
price?: number;
|
|
55
|
+
currency?: string;
|
|
56
|
+
unit?: string;
|
|
57
|
+
url?: string;
|
|
58
|
+
image?: string;
|
|
59
|
+
tags: string[];
|
|
60
|
+
attributes: Record<string, string>;
|
|
61
|
+
}
|
|
62
|
+
interface CatalogRowError {
|
|
63
|
+
/** 1-based, counting the header as row 1 in CSV, or the array index + 1 in JSON. */
|
|
64
|
+
row: number;
|
|
65
|
+
message: string;
|
|
66
|
+
}
|
|
67
|
+
interface ParsedCatalog {
|
|
68
|
+
items: CatalogItem[];
|
|
69
|
+
errors: CatalogRowError[];
|
|
70
|
+
}
|
|
71
|
+
/**
|
|
72
|
+
* Parses CSV per RFC 4180: quoted fields, doubled quotes, commas and newlines
|
|
73
|
+
* inside quotes, CRLF. The separator (`,` or `;`, which Spanish Excel uses) is
|
|
74
|
+
* whichever appears more in the header line.
|
|
75
|
+
*/
|
|
76
|
+
declare function parseCsv(text: string): string[][];
|
|
77
|
+
/**
|
|
78
|
+
* Reads a catalog file. `format` defaults to JSON when the text starts with
|
|
79
|
+
* `[`, CSV otherwise. Rows with errors are skipped and listed, so the owner can
|
|
80
|
+
* fix them; a duplicated id keeps its first row.
|
|
81
|
+
*/
|
|
82
|
+
declare function parseCatalog(text: string, options?: {
|
|
83
|
+
format?: 'csv' | 'json';
|
|
84
|
+
}): ParsedCatalog;
|
|
85
|
+
|
|
86
|
+
export { type CatalogItem, type CatalogRowError, IngestError, type IngestErrorCode, type IngestOptions, type KnowledgeDoc, type ParsedCatalog, type UploadedFile, htmlToText, parseCatalog, parseCsv, safeFileName, toKnowledgeDoc };
|
package/dist/index.js
ADDED
|
@@ -0,0 +1,312 @@
|
|
|
1
|
+
// src/errors.ts
|
|
2
|
+
var IngestError = class extends Error {
|
|
3
|
+
code;
|
|
4
|
+
constructor(code, message, options) {
|
|
5
|
+
super(message, options);
|
|
6
|
+
this.name = "IngestError";
|
|
7
|
+
this.code = code;
|
|
8
|
+
}
|
|
9
|
+
};
|
|
10
|
+
|
|
11
|
+
// src/html.ts
|
|
12
|
+
var BLOCK_TAGS = /<\/?(p|div|section|article|header|footer|main|aside|nav|h[1-6]|li|ul|ol|tr|table|br|hr|blockquote|pre|dt|dd)\b[^>]*>/gi;
|
|
13
|
+
var ENTITIES = {
|
|
14
|
+
amp: "&",
|
|
15
|
+
lt: "<",
|
|
16
|
+
gt: ">",
|
|
17
|
+
quot: '"',
|
|
18
|
+
apos: "'",
|
|
19
|
+
nbsp: " "
|
|
20
|
+
};
|
|
21
|
+
function decodeEntities(text) {
|
|
22
|
+
return text.replace(/&(#x[0-9a-f]+|#\d+|[a-z]+);/gi, (match, entity) => {
|
|
23
|
+
if (entity[0] === "#") {
|
|
24
|
+
const code = entity[1]?.toLowerCase() === "x" ? parseInt(entity.slice(2), 16) : Number(entity.slice(1));
|
|
25
|
+
return Number.isFinite(code) && code > 0 && code <= 1114111 ? String.fromCodePoint(code) : match;
|
|
26
|
+
}
|
|
27
|
+
return ENTITIES[entity.toLowerCase()] ?? match;
|
|
28
|
+
});
|
|
29
|
+
}
|
|
30
|
+
function htmlToText(html) {
|
|
31
|
+
return decodeEntities(
|
|
32
|
+
html.replace(/<!--[\s\S]*?-->/g, "").replace(/<(script|style|noscript|template|svg)\b[\s\S]*?<\/\1\s*>/gi, "").replace(BLOCK_TAGS, "\n").replace(/<[^>]+>/g, "")
|
|
33
|
+
).replace(/[ \t\f\v\u00a0]+/g, " ").replace(/ *\n */g, "\n").replace(/\n{3,}/g, "\n\n").trim();
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
// src/ingest.ts
|
|
37
|
+
var BY_EXTENSION = {
|
|
38
|
+
md: "markdown",
|
|
39
|
+
markdown: "markdown",
|
|
40
|
+
txt: "text",
|
|
41
|
+
html: "html",
|
|
42
|
+
htm: "html",
|
|
43
|
+
pdf: "pdf",
|
|
44
|
+
docx: "docx"
|
|
45
|
+
};
|
|
46
|
+
var BY_MIME = {
|
|
47
|
+
"text/markdown": "markdown",
|
|
48
|
+
"text/plain": "text",
|
|
49
|
+
"text/html": "html",
|
|
50
|
+
"application/pdf": "pdf",
|
|
51
|
+
"application/vnd.openxmlformats-officedocument.wordprocessingml.document": "docx"
|
|
52
|
+
};
|
|
53
|
+
function kindOf(file) {
|
|
54
|
+
const extension = /\.([a-z0-9]+)$/i.exec(file.name)?.[1]?.toLowerCase();
|
|
55
|
+
return extension && BY_EXTENSION[extension] || BY_MIME[file.type?.split(";")[0]?.trim() ?? ""];
|
|
56
|
+
}
|
|
57
|
+
function safeFileName(name) {
|
|
58
|
+
const base = name.replace(/\.[a-z0-9]+$/i, "");
|
|
59
|
+
const slug2 = base.normalize("NFKD").replace(/[̀-ͯ]/g, "").toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 80).replace(/-+$/, "");
|
|
60
|
+
return `${slug2 || "document"}.md`;
|
|
61
|
+
}
|
|
62
|
+
async function toKnowledgeDoc(file, options = {}) {
|
|
63
|
+
const { maxBytes = 10 * 1024 * 1024 } = options;
|
|
64
|
+
if (file.bytes.byteLength > maxBytes) {
|
|
65
|
+
throw new IngestError("too_large", `${file.name} is over ${maxBytes} bytes`);
|
|
66
|
+
}
|
|
67
|
+
const kind = kindOf(file);
|
|
68
|
+
if (!kind) {
|
|
69
|
+
throw new IngestError("unsupported_type", `${file.name}: use .md, .txt, .html, .pdf or .docx`);
|
|
70
|
+
}
|
|
71
|
+
const raw = await extract(kind, file.bytes);
|
|
72
|
+
const content = raw.replace(/\r\n?/g, "\n").replace(/\n{3,}/g, "\n\n").trim();
|
|
73
|
+
if (content === "") {
|
|
74
|
+
const hint = kind === "pdf" ? " (a scanned PDF has no text layer; export it with text)" : "";
|
|
75
|
+
throw new IngestError("empty", `${file.name} has no text${hint}`);
|
|
76
|
+
}
|
|
77
|
+
return { file: safeFileName(file.name), content };
|
|
78
|
+
}
|
|
79
|
+
async function extract(kind, bytes) {
|
|
80
|
+
switch (kind) {
|
|
81
|
+
case "markdown":
|
|
82
|
+
case "text":
|
|
83
|
+
return decode(bytes);
|
|
84
|
+
case "html":
|
|
85
|
+
return htmlToText(decode(bytes));
|
|
86
|
+
case "pdf": {
|
|
87
|
+
const { extractText } = await load("unpdf", () => import("unpdf"));
|
|
88
|
+
const { text } = await extractText(new Uint8Array(bytes), { mergePages: true });
|
|
89
|
+
return text;
|
|
90
|
+
}
|
|
91
|
+
case "docx": {
|
|
92
|
+
const mammoth = await load("mammoth", () => import("mammoth"));
|
|
93
|
+
const { value } = await (mammoth.default ?? mammoth).extractRawText({
|
|
94
|
+
buffer: Buffer.from(bytes)
|
|
95
|
+
});
|
|
96
|
+
return value;
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
function decode(bytes) {
|
|
101
|
+
return new TextDecoder("utf-8").decode(bytes);
|
|
102
|
+
}
|
|
103
|
+
async function load(name, importer) {
|
|
104
|
+
try {
|
|
105
|
+
return await importer();
|
|
106
|
+
} catch (error) {
|
|
107
|
+
throw new IngestError(
|
|
108
|
+
"missing_parser",
|
|
109
|
+
`Install ${name} to read these files: pnpm add ${name}`,
|
|
110
|
+
{
|
|
111
|
+
cause: error
|
|
112
|
+
}
|
|
113
|
+
);
|
|
114
|
+
}
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
// src/catalog.ts
|
|
118
|
+
var ALIASES = {
|
|
119
|
+
id: "id",
|
|
120
|
+
sku: "id",
|
|
121
|
+
codigo: "id",
|
|
122
|
+
code: "id",
|
|
123
|
+
ref: "id",
|
|
124
|
+
referencia: "id",
|
|
125
|
+
name: "name",
|
|
126
|
+
nombre: "name",
|
|
127
|
+
producto: "name",
|
|
128
|
+
product: "name",
|
|
129
|
+
title: "name",
|
|
130
|
+
titulo: "name",
|
|
131
|
+
description: "description",
|
|
132
|
+
descripcion: "description",
|
|
133
|
+
price: "price",
|
|
134
|
+
precio: "price",
|
|
135
|
+
currency: "currency",
|
|
136
|
+
moneda: "currency",
|
|
137
|
+
unit: "unit",
|
|
138
|
+
unidad: "unit",
|
|
139
|
+
url: "url",
|
|
140
|
+
link: "url",
|
|
141
|
+
enlace: "url",
|
|
142
|
+
pagina: "url",
|
|
143
|
+
image: "image",
|
|
144
|
+
imagen: "image",
|
|
145
|
+
foto: "image",
|
|
146
|
+
photo: "image",
|
|
147
|
+
tags: "tags",
|
|
148
|
+
etiquetas: "tags",
|
|
149
|
+
categoria: "tags",
|
|
150
|
+
category: "tags"
|
|
151
|
+
};
|
|
152
|
+
var normalizeKey = (key) => key.normalize("NFKD").replace(/[̀-ͯ]/g, "").trim().toLowerCase();
|
|
153
|
+
var slug = (text) => text.normalize("NFKD").replace(/[̀-ͯ]/g, "").toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 80);
|
|
154
|
+
function parseCsv(text) {
|
|
155
|
+
const input = text.replace(/^\uFEFF/, "");
|
|
156
|
+
const firstLine = input.slice(
|
|
157
|
+
0,
|
|
158
|
+
input.search(/\r?\n/) === -1 ? void 0 : input.search(/\r?\n/)
|
|
159
|
+
);
|
|
160
|
+
const separator = (firstLine.match(/;/g)?.length ?? 0) > (firstLine.match(/,/g)?.length ?? 0) ? ";" : ",";
|
|
161
|
+
const rows = [];
|
|
162
|
+
let row = [];
|
|
163
|
+
let field = "";
|
|
164
|
+
let quoted = false;
|
|
165
|
+
for (let i = 0; i < input.length; i++) {
|
|
166
|
+
const char = input[i];
|
|
167
|
+
if (quoted) {
|
|
168
|
+
if (char === '"' && input[i + 1] === '"') {
|
|
169
|
+
field += '"';
|
|
170
|
+
i++;
|
|
171
|
+
} else if (char === '"') {
|
|
172
|
+
quoted = false;
|
|
173
|
+
} else {
|
|
174
|
+
field += char;
|
|
175
|
+
}
|
|
176
|
+
} else if (char === '"' && field === "") {
|
|
177
|
+
quoted = true;
|
|
178
|
+
} else if (char === separator) {
|
|
179
|
+
row.push(field);
|
|
180
|
+
field = "";
|
|
181
|
+
} else if (char === "\n" || char === "\r") {
|
|
182
|
+
if (char === "\r" && input[i + 1] === "\n") i++;
|
|
183
|
+
row.push(field);
|
|
184
|
+
rows.push(row);
|
|
185
|
+
row = [];
|
|
186
|
+
field = "";
|
|
187
|
+
} else {
|
|
188
|
+
field += char;
|
|
189
|
+
}
|
|
190
|
+
}
|
|
191
|
+
if (field !== "" || row.length > 0) {
|
|
192
|
+
row.push(field);
|
|
193
|
+
rows.push(row);
|
|
194
|
+
}
|
|
195
|
+
return rows;
|
|
196
|
+
}
|
|
197
|
+
var isBlank = (cells) => cells.every((cell) => cell.trim() === "");
|
|
198
|
+
var isLink = (value) => {
|
|
199
|
+
if (/^\/(?![/\\])/.test(value)) return !/\s/.test(value);
|
|
200
|
+
try {
|
|
201
|
+
return ["http:", "https:"].includes(new URL(value).protocol);
|
|
202
|
+
} catch {
|
|
203
|
+
return false;
|
|
204
|
+
}
|
|
205
|
+
};
|
|
206
|
+
function parsePrice(value) {
|
|
207
|
+
let text = value.replace(/[^\d.,-]/g, "");
|
|
208
|
+
if (!text) return void 0;
|
|
209
|
+
const comma = text.lastIndexOf(",");
|
|
210
|
+
const dot = text.lastIndexOf(".");
|
|
211
|
+
if (comma > dot) text = text.replace(/\./g, "").replace(",", ".");
|
|
212
|
+
else text = text.replace(/,/g, "");
|
|
213
|
+
const number = Number(text);
|
|
214
|
+
return Number.isFinite(number) && number >= 0 ? number : void 0;
|
|
215
|
+
}
|
|
216
|
+
function toItem(record) {
|
|
217
|
+
const item = { id: "", name: "", tags: [], attributes: {} };
|
|
218
|
+
for (const [rawKey, rawValue] of Object.entries(record)) {
|
|
219
|
+
if (rawValue === null || rawValue === void 0) continue;
|
|
220
|
+
const field = ALIASES[normalizeKey(rawKey)];
|
|
221
|
+
if (field === "tags") {
|
|
222
|
+
const values = Array.isArray(rawValue) ? rawValue.map(String) : String(rawValue).split(/[|;]/);
|
|
223
|
+
item.tags = values.map((tag) => tag.trim()).filter(Boolean);
|
|
224
|
+
continue;
|
|
225
|
+
}
|
|
226
|
+
if (rawKey === "attributes" && typeof rawValue === "object" && !Array.isArray(rawValue)) {
|
|
227
|
+
for (const [key, value2] of Object.entries(rawValue)) {
|
|
228
|
+
if (String(value2).trim()) item.attributes[key] = String(value2).trim();
|
|
229
|
+
}
|
|
230
|
+
continue;
|
|
231
|
+
}
|
|
232
|
+
const value = typeof rawValue === "number" ? String(rawValue) : String(rawValue).trim();
|
|
233
|
+
if (!value) continue;
|
|
234
|
+
if (!field) {
|
|
235
|
+
item.attributes[rawKey.trim()] = value;
|
|
236
|
+
} else if (field === "price") {
|
|
237
|
+
const price = typeof rawValue === "number" ? rawValue : parsePrice(value);
|
|
238
|
+
if (price === void 0) return `price "${value}" is not a number`;
|
|
239
|
+
item.price = price;
|
|
240
|
+
} else if ((field === "url" || field === "image") && !isLink(value)) {
|
|
241
|
+
return `${field} "${value}" is not an http(s) link or a path starting with /`;
|
|
242
|
+
} else {
|
|
243
|
+
item[field] = value;
|
|
244
|
+
}
|
|
245
|
+
}
|
|
246
|
+
if (!item.name) return "name is missing";
|
|
247
|
+
item.id ||= slug(item.name);
|
|
248
|
+
if (!item.id) return "id is missing and the name has no letters or digits";
|
|
249
|
+
if (item.currency && !/^[A-Za-z]{3}$/.test(item.currency)) {
|
|
250
|
+
return `currency "${item.currency}" should be a 3-letter code like USD`;
|
|
251
|
+
}
|
|
252
|
+
return item;
|
|
253
|
+
}
|
|
254
|
+
function parseCatalog(text, options = {}) {
|
|
255
|
+
const format = options.format ?? (text.trimStart().startsWith("[") ? "json" : "csv");
|
|
256
|
+
const records = [];
|
|
257
|
+
const errors = [];
|
|
258
|
+
if (format === "json") {
|
|
259
|
+
let parsed;
|
|
260
|
+
try {
|
|
261
|
+
parsed = JSON.parse(text);
|
|
262
|
+
} catch {
|
|
263
|
+
return { items: [], errors: [{ row: 0, message: "The file is not valid JSON" }] };
|
|
264
|
+
}
|
|
265
|
+
if (!Array.isArray(parsed)) {
|
|
266
|
+
return { items: [], errors: [{ row: 0, message: "The JSON must be an array of products" }] };
|
|
267
|
+
}
|
|
268
|
+
parsed.forEach((record, index) => {
|
|
269
|
+
if (record && typeof record === "object" && !Array.isArray(record)) {
|
|
270
|
+
records.push({ row: index + 1, record });
|
|
271
|
+
} else {
|
|
272
|
+
errors.push({ row: index + 1, message: "not an object" });
|
|
273
|
+
}
|
|
274
|
+
});
|
|
275
|
+
} else {
|
|
276
|
+
const [header, ...rows] = parseCsv(text);
|
|
277
|
+
if (!header || isBlank(header)) {
|
|
278
|
+
return { items: [], errors: [{ row: 0, message: "The file is empty" }] };
|
|
279
|
+
}
|
|
280
|
+
rows.forEach((cells, index) => {
|
|
281
|
+
if (isBlank(cells)) return;
|
|
282
|
+
const record = {};
|
|
283
|
+
header.forEach((column, i) => {
|
|
284
|
+
if (column.trim()) record[column.trim()] = cells[i] ?? "";
|
|
285
|
+
});
|
|
286
|
+
records.push({ row: index + 2, record });
|
|
287
|
+
});
|
|
288
|
+
}
|
|
289
|
+
const items = [];
|
|
290
|
+
const seen = /* @__PURE__ */ new Set();
|
|
291
|
+
for (const { row, record } of records) {
|
|
292
|
+
const result = toItem(record);
|
|
293
|
+
if (typeof result === "string") {
|
|
294
|
+
errors.push({ row, message: result });
|
|
295
|
+
} else if (seen.has(result.id)) {
|
|
296
|
+
errors.push({ row, message: `duplicate id "${result.id}", kept the first one` });
|
|
297
|
+
} else {
|
|
298
|
+
seen.add(result.id);
|
|
299
|
+
items.push(result);
|
|
300
|
+
}
|
|
301
|
+
}
|
|
302
|
+
return { items, errors };
|
|
303
|
+
}
|
|
304
|
+
export {
|
|
305
|
+
IngestError,
|
|
306
|
+
htmlToText,
|
|
307
|
+
parseCatalog,
|
|
308
|
+
parseCsv,
|
|
309
|
+
safeFileName,
|
|
310
|
+
toKnowledgeDoc
|
|
311
|
+
};
|
|
312
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"sources":["../src/errors.ts","../src/html.ts","../src/ingest.ts","../src/catalog.ts"],"sourcesContent":["export type IngestErrorCode = 'unsupported_type' | 'too_large' | 'empty' | 'missing_parser';\n\n/** Why a file couldn't become knowledge. `message` is meant for the site owner, in English. */\nexport class IngestError extends Error {\n readonly code: IngestErrorCode;\n\n constructor(code: IngestErrorCode, message: string, options?: { cause?: unknown }) {\n super(message, options);\n this.name = 'IngestError';\n this.code = code;\n }\n}\n","const BLOCK_TAGS =\n /<\\/?(p|div|section|article|header|footer|main|aside|nav|h[1-6]|li|ul|ol|tr|table|br|hr|blockquote|pre|dt|dd)\\b[^>]*>/gi;\n\nconst ENTITIES: Record<string, string> = {\n amp: '&',\n lt: '<',\n gt: '>',\n quot: '\"',\n apos: \"'\",\n nbsp: ' ',\n};\n\nfunction decodeEntities(text: string): string {\n return text.replace(/&(#x[0-9a-f]+|#\\d+|[a-z]+);/gi, (match, entity: string) => {\n if (entity[0] === '#') {\n const code =\n entity[1]?.toLowerCase() === 'x' ? parseInt(entity.slice(2), 16) : Number(entity.slice(1));\n return Number.isFinite(code) && code > 0 && code <= 0x10ffff\n ? String.fromCodePoint(code)\n : match;\n }\n return ENTITIES[entity.toLowerCase()] ?? match;\n });\n}\n\n/**\n * Plain text from an HTML page: scripts, styles and comments dropped, block\n * elements on their own lines. Not a sanitizer: the output is only ever text.\n */\nexport function htmlToText(html: string): string {\n return decodeEntities(\n html\n .replace(/<!--[\\s\\S]*?-->/g, '')\n .replace(/<(script|style|noscript|template|svg)\\b[\\s\\S]*?<\\/\\1\\s*>/gi, '')\n .replace(BLOCK_TAGS, '\\n')\n .replace(/<[^>]+>/g, ''),\n )\n .replace(/[ \\t\\f\\v\\u00a0]+/g, ' ')\n .replace(/ *\\n */g, '\\n')\n .replace(/\\n{3,}/g, '\\n\\n')\n .trim();\n}\n","import { IngestError } from './errors.js';\nimport { htmlToText } from './html.js';\n\n/** The same shape as ai-chat-core's `KnowledgeDoc`, so the output can be handed to a source. */\nexport interface KnowledgeDoc {\n file: string;\n content: string;\n}\n\nexport interface UploadedFile {\n /** The original file name; its extension picks the parser. */\n name: string;\n /** MIME type, used when the name has no known extension. */\n type?: string;\n bytes: Uint8Array;\n}\n\nexport interface IngestOptions {\n /** Default 10 MB. */\n maxBytes?: number;\n}\n\ntype Kind = 'markdown' | 'text' | 'html' | 'pdf' | 'docx';\n\nconst BY_EXTENSION: Record<string, Kind> = {\n md: 'markdown',\n markdown: 'markdown',\n txt: 'text',\n html: 'html',\n htm: 'html',\n pdf: 'pdf',\n docx: 'docx',\n};\n\nconst BY_MIME: Record<string, Kind> = {\n 'text/markdown': 'markdown',\n 'text/plain': 'text',\n 'text/html': 'html',\n 'application/pdf': 'pdf',\n 'application/vnd.openxmlformats-officedocument.wordprocessingml.document': 'docx',\n};\n\nfunction kindOf(file: UploadedFile): Kind | undefined {\n const extension = /\\.([a-z0-9]+)$/i.exec(file.name)?.[1]?.toLowerCase();\n return (extension && BY_EXTENSION[extension]) || BY_MIME[file.type?.split(';')[0]?.trim() ?? ''];\n}\n\n/**\n * A file name that is safe to write anywhere: lowercase ASCII words joined by\n * dashes, always ending in `.md`. Path separators, dots and accents can't\n * survive it, so `../../etc/passwd` becomes `etc-passwd.md`.\n */\nexport function safeFileName(name: string): string {\n const base = name.replace(/\\.[a-z0-9]+$/i, '');\n const slug = base\n .normalize('NFKD')\n .replace(/[̀-ͯ]/g, '')\n .toLowerCase()\n .replace(/[^a-z0-9]+/g, '-')\n .replace(/^-+|-+$/g, '')\n .slice(0, 80)\n .replace(/-+$/, '');\n return `${slug || 'document'}.md`;\n}\n\n/**\n * Converts an uploaded file to a knowledge document. Run it when the owner\n * uploads, not during a chat: PDF and DOCX parsing is slow, and storing the\n * text keeps the bot's prompt fast and identical between requests.\n */\nexport async function toKnowledgeDoc(\n file: UploadedFile,\n options: IngestOptions = {},\n): Promise<KnowledgeDoc> {\n const { maxBytes = 10 * 1024 * 1024 } = options;\n if (file.bytes.byteLength > maxBytes) {\n throw new IngestError('too_large', `${file.name} is over ${maxBytes} bytes`);\n }\n const kind = kindOf(file);\n if (!kind) {\n throw new IngestError('unsupported_type', `${file.name}: use .md, .txt, .html, .pdf or .docx`);\n }\n\n const raw = await extract(kind, file.bytes);\n const content = raw\n .replace(/\\r\\n?/g, '\\n')\n .replace(/\\n{3,}/g, '\\n\\n')\n .trim();\n if (content === '') {\n const hint = kind === 'pdf' ? ' (a scanned PDF has no text layer; export it with text)' : '';\n throw new IngestError('empty', `${file.name} has no text${hint}`);\n }\n return { file: safeFileName(file.name), content };\n}\n\nasync function extract(kind: Kind, bytes: Uint8Array): Promise<string> {\n switch (kind) {\n case 'markdown':\n case 'text':\n return decode(bytes);\n case 'html':\n return htmlToText(decode(bytes));\n case 'pdf': {\n const { extractText } = await load('unpdf', () => import('unpdf'));\n // unpdf may transfer the buffer to a worker, so give it a copy.\n const { text } = await extractText(new Uint8Array(bytes), { mergePages: true });\n return text;\n }\n case 'docx': {\n const mammoth = await load('mammoth', () => import('mammoth'));\n const { value } = await (mammoth.default ?? mammoth).extractRawText({\n buffer: Buffer.from(bytes),\n });\n return value;\n }\n }\n}\n\n/** UTF-8 text. TextDecoder drops a leading byte-order mark by itself. */\nfunction decode(bytes: Uint8Array): string {\n return new TextDecoder('utf-8').decode(bytes);\n}\n\nasync function load<T>(name: string, importer: () => Promise<T>): Promise<T> {\n try {\n return await importer();\n } catch (error) {\n throw new IngestError(\n 'missing_parser',\n `Install ${name} to read these files: pnpm add ${name}`,\n {\n cause: error,\n },\n );\n }\n}\n","/**\n * Catalog import: a CSV or JSON file from the site owner becomes catalog items\n * in the shape ai-chat-core expects. Bad rows are reported, the rest import.\n */\n\n/** The same shape as ai-chat-core's `CatalogItemInput`; core validates it again on load. */\nexport interface CatalogItem {\n id: string;\n name: string;\n description?: string;\n price?: number;\n currency?: string;\n unit?: string;\n url?: string;\n image?: string;\n tags: string[];\n attributes: Record<string, string>;\n}\n\nexport interface CatalogRowError {\n /** 1-based, counting the header as row 1 in CSV, or the array index + 1 in JSON. */\n row: number;\n message: string;\n}\n\nexport interface ParsedCatalog {\n items: CatalogItem[];\n errors: CatalogRowError[];\n}\n\ntype Field = Exclude<keyof CatalogItem, 'attributes'>;\n\n/** Column names site owners write, in Spanish and English, accents and case ignored. */\nconst ALIASES: Record<string, Field> = {\n id: 'id',\n sku: 'id',\n codigo: 'id',\n code: 'id',\n ref: 'id',\n referencia: 'id',\n name: 'name',\n nombre: 'name',\n producto: 'name',\n product: 'name',\n title: 'name',\n titulo: 'name',\n description: 'description',\n descripcion: 'description',\n price: 'price',\n precio: 'price',\n currency: 'currency',\n moneda: 'currency',\n unit: 'unit',\n unidad: 'unit',\n url: 'url',\n link: 'url',\n enlace: 'url',\n pagina: 'url',\n image: 'image',\n imagen: 'image',\n foto: 'image',\n photo: 'image',\n tags: 'tags',\n etiquetas: 'tags',\n categoria: 'tags',\n category: 'tags',\n};\n\nconst normalizeKey = (key: string) =>\n key.normalize('NFKD').replace(/[̀-ͯ]/g, '').trim().toLowerCase();\n\nconst slug = (text: string) =>\n text\n .normalize('NFKD')\n .replace(/[̀-ͯ]/g, '')\n .toLowerCase()\n .replace(/[^a-z0-9]+/g, '-')\n .replace(/^-+|-+$/g, '')\n .slice(0, 80);\n\n/**\n * Parses CSV per RFC 4180: quoted fields, doubled quotes, commas and newlines\n * inside quotes, CRLF. The separator (`,` or `;`, which Spanish Excel uses) is\n * whichever appears more in the header line.\n */\nexport function parseCsv(text: string): string[][] {\n const input = text.replace(/^\\uFEFF/, '');\n const firstLine = input.slice(\n 0,\n input.search(/\\r?\\n/) === -1 ? undefined : input.search(/\\r?\\n/),\n );\n const separator =\n (firstLine.match(/;/g)?.length ?? 0) > (firstLine.match(/,/g)?.length ?? 0) ? ';' : ',';\n\n const rows: string[][] = [];\n let row: string[] = [];\n let field = '';\n let quoted = false;\n for (let i = 0; i < input.length; i++) {\n const char = input[i];\n if (quoted) {\n if (char === '\"' && input[i + 1] === '\"') {\n field += '\"';\n i++;\n } else if (char === '\"') {\n quoted = false;\n } else {\n field += char;\n }\n } else if (char === '\"' && field === '') {\n quoted = true;\n } else if (char === separator) {\n row.push(field);\n field = '';\n } else if (char === '\\n' || char === '\\r') {\n if (char === '\\r' && input[i + 1] === '\\n') i++;\n row.push(field);\n rows.push(row);\n row = [];\n field = '';\n } else {\n field += char;\n }\n }\n if (field !== '' || row.length > 0) {\n row.push(field);\n rows.push(row);\n }\n return rows;\n}\n\nconst isBlank = (cells: string[]) => cells.every((cell) => cell.trim() === '');\n\n/** An absolute http(s) URL, or a root-relative path on the site (`/images/x.webp`). */\nconst isLink = (value: string) => {\n if (/^\\/(?![/\\\\])/.test(value)) return !/\\s/.test(value);\n try {\n return ['http:', 'https:'].includes(new URL(value).protocol);\n } catch {\n return false;\n }\n};\n\n/** A price as owners write it: \"0,25\", \"1.234,50\", \"$ 12.5\", \"12.50 USD\". */\nfunction parsePrice(value: string): number | undefined {\n let text = value.replace(/[^\\d.,-]/g, '');\n if (!text) return undefined;\n const comma = text.lastIndexOf(',');\n const dot = text.lastIndexOf('.');\n // The rightmost separator is the decimal one; the other groups thousands.\n if (comma > dot) text = text.replace(/\\./g, '').replace(',', '.');\n else text = text.replace(/,/g, '');\n const number = Number(text);\n return Number.isFinite(number) && number >= 0 ? number : undefined;\n}\n\n/** One record (CSV row or JSON object) into an item, or the reason it can't be one. */\nfunction toItem(record: Record<string, unknown>): CatalogItem | string {\n const item: CatalogItem = { id: '', name: '', tags: [], attributes: {} };\n for (const [rawKey, rawValue] of Object.entries(record)) {\n if (rawValue === null || rawValue === undefined) continue;\n const field = ALIASES[normalizeKey(rawKey)];\n if (field === 'tags') {\n const values = Array.isArray(rawValue)\n ? rawValue.map(String)\n : String(rawValue).split(/[|;]/);\n item.tags = values.map((tag) => tag.trim()).filter(Boolean);\n continue;\n }\n if (rawKey === 'attributes' && typeof rawValue === 'object' && !Array.isArray(rawValue)) {\n for (const [key, value] of Object.entries(rawValue)) {\n if (String(value).trim()) item.attributes[key] = String(value).trim();\n }\n continue;\n }\n const value = typeof rawValue === 'number' ? String(rawValue) : String(rawValue).trim();\n if (!value) continue;\n if (!field) {\n item.attributes[rawKey.trim()] = value;\n } else if (field === 'price') {\n const price = typeof rawValue === 'number' ? rawValue : parsePrice(value);\n if (price === undefined) return `price \"${value}\" is not a number`;\n item.price = price;\n } else if ((field === 'url' || field === 'image') && !isLink(value)) {\n return `${field} \"${value}\" is not an http(s) link or a path starting with /`;\n } else {\n item[field] = value;\n }\n }\n if (!item.name) return 'name is missing';\n item.id ||= slug(item.name);\n if (!item.id) return 'id is missing and the name has no letters or digits';\n if (item.currency && !/^[A-Za-z]{3}$/.test(item.currency)) {\n return `currency \"${item.currency}\" should be a 3-letter code like USD`;\n }\n return item;\n}\n\n/**\n * Reads a catalog file. `format` defaults to JSON when the text starts with\n * `[`, CSV otherwise. Rows with errors are skipped and listed, so the owner can\n * fix them; a duplicated id keeps its first row.\n */\nexport function parseCatalog(\n text: string,\n options: { format?: 'csv' | 'json' } = {},\n): ParsedCatalog {\n const format = options.format ?? (text.trimStart().startsWith('[') ? 'json' : 'csv');\n const records: { row: number; record: Record<string, unknown> }[] = [];\n const errors: CatalogRowError[] = [];\n\n if (format === 'json') {\n let parsed: unknown;\n try {\n parsed = JSON.parse(text);\n } catch {\n return { items: [], errors: [{ row: 0, message: 'The file is not valid JSON' }] };\n }\n if (!Array.isArray(parsed)) {\n return { items: [], errors: [{ row: 0, message: 'The JSON must be an array of products' }] };\n }\n parsed.forEach((record, index) => {\n if (record && typeof record === 'object' && !Array.isArray(record)) {\n records.push({ row: index + 1, record: record as Record<string, unknown> });\n } else {\n errors.push({ row: index + 1, message: 'not an object' });\n }\n });\n } else {\n const [header, ...rows] = parseCsv(text);\n if (!header || isBlank(header)) {\n return { items: [], errors: [{ row: 0, message: 'The file is empty' }] };\n }\n rows.forEach((cells, index) => {\n // Blank rows are skipped but still counted, so row numbers match the spreadsheet.\n if (isBlank(cells)) return;\n const record: Record<string, string> = {};\n header.forEach((column, i) => {\n if (column.trim()) record[column.trim()] = cells[i] ?? '';\n });\n records.push({ row: index + 2, record });\n });\n }\n\n const items: CatalogItem[] = [];\n const seen = new Set<string>();\n for (const { row, record } of records) {\n const result = toItem(record);\n if (typeof result === 'string') {\n errors.push({ row, message: result });\n } else if (seen.has(result.id)) {\n errors.push({ row, message: `duplicate id \"${result.id}\", kept the first one` });\n } else {\n seen.add(result.id);\n items.push(result);\n }\n }\n return { items, errors };\n}\n"],"mappings":";AAGO,IAAM,cAAN,cAA0B,MAAM;AAAA,EAC5B;AAAA,EAET,YAAY,MAAuB,SAAiB,SAA+B;AACjF,UAAM,SAAS,OAAO;AACtB,SAAK,OAAO;AACZ,SAAK,OAAO;AAAA,EACd;AACF;;;ACXA,IAAM,aACJ;AAEF,IAAM,WAAmC;AAAA,EACvC,KAAK;AAAA,EACL,IAAI;AAAA,EACJ,IAAI;AAAA,EACJ,MAAM;AAAA,EACN,MAAM;AAAA,EACN,MAAM;AACR;AAEA,SAAS,eAAe,MAAsB;AAC5C,SAAO,KAAK,QAAQ,iCAAiC,CAAC,OAAO,WAAmB;AAC9E,QAAI,OAAO,CAAC,MAAM,KAAK;AACrB,YAAM,OACJ,OAAO,CAAC,GAAG,YAAY,MAAM,MAAM,SAAS,OAAO,MAAM,CAAC,GAAG,EAAE,IAAI,OAAO,OAAO,MAAM,CAAC,CAAC;AAC3F,aAAO,OAAO,SAAS,IAAI,KAAK,OAAO,KAAK,QAAQ,UAChD,OAAO,cAAc,IAAI,IACzB;AAAA,IACN;AACA,WAAO,SAAS,OAAO,YAAY,CAAC,KAAK;AAAA,EAC3C,CAAC;AACH;AAMO,SAAS,WAAW,MAAsB;AAC/C,SAAO;AAAA,IACL,KACG,QAAQ,oBAAoB,EAAE,EAC9B,QAAQ,8DAA8D,EAAE,EACxE,QAAQ,YAAY,IAAI,EACxB,QAAQ,YAAY,EAAE;AAAA,EAC3B,EACG,QAAQ,qBAAqB,GAAG,EAChC,QAAQ,WAAW,IAAI,EACvB,QAAQ,WAAW,MAAM,EACzB,KAAK;AACV;;;ACjBA,IAAM,eAAqC;AAAA,EACzC,IAAI;AAAA,EACJ,UAAU;AAAA,EACV,KAAK;AAAA,EACL,MAAM;AAAA,EACN,KAAK;AAAA,EACL,KAAK;AAAA,EACL,MAAM;AACR;AAEA,IAAM,UAAgC;AAAA,EACpC,iBAAiB;AAAA,EACjB,cAAc;AAAA,EACd,aAAa;AAAA,EACb,mBAAmB;AAAA,EACnB,2EAA2E;AAC7E;AAEA,SAAS,OAAO,MAAsC;AACpD,QAAM,YAAY,kBAAkB,KAAK,KAAK,IAAI,IAAI,CAAC,GAAG,YAAY;AACtE,SAAQ,aAAa,aAAa,SAAS,KAAM,QAAQ,KAAK,MAAM,MAAM,GAAG,EAAE,CAAC,GAAG,KAAK,KAAK,EAAE;AACjG;AAOO,SAAS,aAAa,MAAsB;AACjD,QAAM,OAAO,KAAK,QAAQ,iBAAiB,EAAE;AAC7C,QAAMA,QAAO,KACV,UAAU,MAAM,EAChB,QAAQ,UAAU,EAAE,EACpB,YAAY,EACZ,QAAQ,eAAe,GAAG,EAC1B,QAAQ,YAAY,EAAE,EACtB,MAAM,GAAG,EAAE,EACX,QAAQ,OAAO,EAAE;AACpB,SAAO,GAAGA,SAAQ,UAAU;AAC9B;AAOA,eAAsB,eACpB,MACA,UAAyB,CAAC,GACH;AACvB,QAAM,EAAE,WAAW,KAAK,OAAO,KAAK,IAAI;AACxC,MAAI,KAAK,MAAM,aAAa,UAAU;AACpC,UAAM,IAAI,YAAY,aAAa,GAAG,KAAK,IAAI,YAAY,QAAQ,QAAQ;AAAA,EAC7E;AACA,QAAM,OAAO,OAAO,IAAI;AACxB,MAAI,CAAC,MAAM;AACT,UAAM,IAAI,YAAY,oBAAoB,GAAG,KAAK,IAAI,uCAAuC;AAAA,EAC/F;AAEA,QAAM,MAAM,MAAM,QAAQ,MAAM,KAAK,KAAK;AAC1C,QAAM,UAAU,IACb,QAAQ,UAAU,IAAI,EACtB,QAAQ,WAAW,MAAM,EACzB,KAAK;AACR,MAAI,YAAY,IAAI;AAClB,UAAM,OAAO,SAAS,QAAQ,4DAA4D;AAC1F,UAAM,IAAI,YAAY,SAAS,GAAG,KAAK,IAAI,eAAe,IAAI,EAAE;AAAA,EAClE;AACA,SAAO,EAAE,MAAM,aAAa,KAAK,IAAI,GAAG,QAAQ;AAClD;AAEA,eAAe,QAAQ,MAAY,OAAoC;AACrE,UAAQ,MAAM;AAAA,IACZ,KAAK;AAAA,IACL,KAAK;AACH,aAAO,OAAO,KAAK;AAAA,IACrB,KAAK;AACH,aAAO,WAAW,OAAO,KAAK,CAAC;AAAA,IACjC,KAAK,OAAO;AACV,YAAM,EAAE,YAAY,IAAI,MAAM,KAAK,SAAS,MAAM,OAAO,OAAO,CAAC;AAEjE,YAAM,EAAE,KAAK,IAAI,MAAM,YAAY,IAAI,WAAW,KAAK,GAAG,EAAE,YAAY,KAAK,CAAC;AAC9E,aAAO;AAAA,IACT;AAAA,IACA,KAAK,QAAQ;AACX,YAAM,UAAU,MAAM,KAAK,WAAW,MAAM,OAAO,SAAS,CAAC;AAC7D,YAAM,EAAE,MAAM,IAAI,OAAO,QAAQ,WAAW,SAAS,eAAe;AAAA,QAClE,QAAQ,OAAO,KAAK,KAAK;AAAA,MAC3B,CAAC;AACD,aAAO;AAAA,IACT;AAAA,EACF;AACF;AAGA,SAAS,OAAO,OAA2B;AACzC,SAAO,IAAI,YAAY,OAAO,EAAE,OAAO,KAAK;AAC9C;AAEA,eAAe,KAAQ,MAAc,UAAwC;AAC3E,MAAI;AACF,WAAO,MAAM,SAAS;AAAA,EACxB,SAAS,OAAO;AACd,UAAM,IAAI;AAAA,MACR;AAAA,MACA,WAAW,IAAI,kCAAkC,IAAI;AAAA,MACrD;AAAA,QACE,OAAO;AAAA,MACT;AAAA,IACF;AAAA,EACF;AACF;;;ACtGA,IAAM,UAAiC;AAAA,EACrC,IAAI;AAAA,EACJ,KAAK;AAAA,EACL,QAAQ;AAAA,EACR,MAAM;AAAA,EACN,KAAK;AAAA,EACL,YAAY;AAAA,EACZ,MAAM;AAAA,EACN,QAAQ;AAAA,EACR,UAAU;AAAA,EACV,SAAS;AAAA,EACT,OAAO;AAAA,EACP,QAAQ;AAAA,EACR,aAAa;AAAA,EACb,aAAa;AAAA,EACb,OAAO;AAAA,EACP,QAAQ;AAAA,EACR,UAAU;AAAA,EACV,QAAQ;AAAA,EACR,MAAM;AAAA,EACN,QAAQ;AAAA,EACR,KAAK;AAAA,EACL,MAAM;AAAA,EACN,QAAQ;AAAA,EACR,QAAQ;AAAA,EACR,OAAO;AAAA,EACP,QAAQ;AAAA,EACR,MAAM;AAAA,EACN,OAAO;AAAA,EACP,MAAM;AAAA,EACN,WAAW;AAAA,EACX,WAAW;AAAA,EACX,UAAU;AACZ;AAEA,IAAM,eAAe,CAAC,QACpB,IAAI,UAAU,MAAM,EAAE,QAAQ,UAAU,EAAE,EAAE,KAAK,EAAE,YAAY;AAEjE,IAAM,OAAO,CAAC,SACZ,KACG,UAAU,MAAM,EAChB,QAAQ,UAAU,EAAE,EACpB,YAAY,EACZ,QAAQ,eAAe,GAAG,EAC1B,QAAQ,YAAY,EAAE,EACtB,MAAM,GAAG,EAAE;AAOT,SAAS,SAAS,MAA0B;AACjD,QAAM,QAAQ,KAAK,QAAQ,WAAW,EAAE;AACxC,QAAM,YAAY,MAAM;AAAA,IACtB;AAAA,IACA,MAAM,OAAO,OAAO,MAAM,KAAK,SAAY,MAAM,OAAO,OAAO;AAAA,EACjE;AACA,QAAM,aACH,UAAU,MAAM,IAAI,GAAG,UAAU,MAAM,UAAU,MAAM,IAAI,GAAG,UAAU,KAAK,MAAM;AAEtF,QAAM,OAAmB,CAAC;AAC1B,MAAI,MAAgB,CAAC;AACrB,MAAI,QAAQ;AACZ,MAAI,SAAS;AACb,WAAS,IAAI,GAAG,IAAI,MAAM,QAAQ,KAAK;AACrC,UAAM,OAAO,MAAM,CAAC;AACpB,QAAI,QAAQ;AACV,UAAI,SAAS,OAAO,MAAM,IAAI,CAAC,MAAM,KAAK;AACxC,iBAAS;AACT;AAAA,MACF,WAAW,SAAS,KAAK;AACvB,iBAAS;AAAA,MACX,OAAO;AACL,iBAAS;AAAA,MACX;AAAA,IACF,WAAW,SAAS,OAAO,UAAU,IAAI;AACvC,eAAS;AAAA,IACX,WAAW,SAAS,WAAW;AAC7B,UAAI,KAAK,KAAK;AACd,cAAQ;AAAA,IACV,WAAW,SAAS,QAAQ,SAAS,MAAM;AACzC,UAAI,SAAS,QAAQ,MAAM,IAAI,CAAC,MAAM,KAAM;AAC5C,UAAI,KAAK,KAAK;AACd,WAAK,KAAK,GAAG;AACb,YAAM,CAAC;AACP,cAAQ;AAAA,IACV,OAAO;AACL,eAAS;AAAA,IACX;AAAA,EACF;AACA,MAAI,UAAU,MAAM,IAAI,SAAS,GAAG;AAClC,QAAI,KAAK,KAAK;AACd,SAAK,KAAK,GAAG;AAAA,EACf;AACA,SAAO;AACT;AAEA,IAAM,UAAU,CAAC,UAAoB,MAAM,MAAM,CAAC,SAAS,KAAK,KAAK,MAAM,EAAE;AAG7E,IAAM,SAAS,CAAC,UAAkB;AAChC,MAAI,eAAe,KAAK,KAAK,EAAG,QAAO,CAAC,KAAK,KAAK,KAAK;AACvD,MAAI;AACF,WAAO,CAAC,SAAS,QAAQ,EAAE,SAAS,IAAI,IAAI,KAAK,EAAE,QAAQ;AAAA,EAC7D,QAAQ;AACN,WAAO;AAAA,EACT;AACF;AAGA,SAAS,WAAW,OAAmC;AACrD,MAAI,OAAO,MAAM,QAAQ,aAAa,EAAE;AACxC,MAAI,CAAC,KAAM,QAAO;AAClB,QAAM,QAAQ,KAAK,YAAY,GAAG;AAClC,QAAM,MAAM,KAAK,YAAY,GAAG;AAEhC,MAAI,QAAQ,IAAK,QAAO,KAAK,QAAQ,OAAO,EAAE,EAAE,QAAQ,KAAK,GAAG;AAAA,MAC3D,QAAO,KAAK,QAAQ,MAAM,EAAE;AACjC,QAAM,SAAS,OAAO,IAAI;AAC1B,SAAO,OAAO,SAAS,MAAM,KAAK,UAAU,IAAI,SAAS;AAC3D;AAGA,SAAS,OAAO,QAAuD;AACrE,QAAM,OAAoB,EAAE,IAAI,IAAI,MAAM,IAAI,MAAM,CAAC,GAAG,YAAY,CAAC,EAAE;AACvE,aAAW,CAAC,QAAQ,QAAQ,KAAK,OAAO,QAAQ,MAAM,GAAG;AACvD,QAAI,aAAa,QAAQ,aAAa,OAAW;AACjD,UAAM,QAAQ,QAAQ,aAAa,MAAM,CAAC;AAC1C,QAAI,UAAU,QAAQ;AACpB,YAAM,SAAS,MAAM,QAAQ,QAAQ,IACjC,SAAS,IAAI,MAAM,IACnB,OAAO,QAAQ,EAAE,MAAM,MAAM;AACjC,WAAK,OAAO,OAAO,IAAI,CAAC,QAAQ,IAAI,KAAK,CAAC,EAAE,OAAO,OAAO;AAC1D;AAAA,IACF;AACA,QAAI,WAAW,gBAAgB,OAAO,aAAa,YAAY,CAAC,MAAM,QAAQ,QAAQ,GAAG;AACvF,iBAAW,CAAC,KAAKC,MAAK,KAAK,OAAO,QAAQ,QAAQ,GAAG;AACnD,YAAI,OAAOA,MAAK,EAAE,KAAK,EAAG,MAAK,WAAW,GAAG,IAAI,OAAOA,MAAK,EAAE,KAAK;AAAA,MACtE;AACA;AAAA,IACF;AACA,UAAM,QAAQ,OAAO,aAAa,WAAW,OAAO,QAAQ,IAAI,OAAO,QAAQ,EAAE,KAAK;AACtF,QAAI,CAAC,MAAO;AACZ,QAAI,CAAC,OAAO;AACV,WAAK,WAAW,OAAO,KAAK,CAAC,IAAI;AAAA,IACnC,WAAW,UAAU,SAAS;AAC5B,YAAM,QAAQ,OAAO,aAAa,WAAW,WAAW,WAAW,KAAK;AACxE,UAAI,UAAU,OAAW,QAAO,UAAU,KAAK;AAC/C,WAAK,QAAQ;AAAA,IACf,YAAY,UAAU,SAAS,UAAU,YAAY,CAAC,OAAO,KAAK,GAAG;AACnE,aAAO,GAAG,KAAK,KAAK,KAAK;AAAA,IAC3B,OAAO;AACL,WAAK,KAAK,IAAI;AAAA,IAChB;AAAA,EACF;AACA,MAAI,CAAC,KAAK,KAAM,QAAO;AACvB,OAAK,OAAO,KAAK,KAAK,IAAI;AAC1B,MAAI,CAAC,KAAK,GAAI,QAAO;AACrB,MAAI,KAAK,YAAY,CAAC,gBAAgB,KAAK,KAAK,QAAQ,GAAG;AACzD,WAAO,aAAa,KAAK,QAAQ;AAAA,EACnC;AACA,SAAO;AACT;AAOO,SAAS,aACd,MACA,UAAuC,CAAC,GACzB;AACf,QAAM,SAAS,QAAQ,WAAW,KAAK,UAAU,EAAE,WAAW,GAAG,IAAI,SAAS;AAC9E,QAAM,UAA8D,CAAC;AACrE,QAAM,SAA4B,CAAC;AAEnC,MAAI,WAAW,QAAQ;AACrB,QAAI;AACJ,QAAI;AACF,eAAS,KAAK,MAAM,IAAI;AAAA,IAC1B,QAAQ;AACN,aAAO,EAAE,OAAO,CAAC,GAAG,QAAQ,CAAC,EAAE,KAAK,GAAG,SAAS,6BAA6B,CAAC,EAAE;AAAA,IAClF;AACA,QAAI,CAAC,MAAM,QAAQ,MAAM,GAAG;AAC1B,aAAO,EAAE,OAAO,CAAC,GAAG,QAAQ,CAAC,EAAE,KAAK,GAAG,SAAS,wCAAwC,CAAC,EAAE;AAAA,IAC7F;AACA,WAAO,QAAQ,CAAC,QAAQ,UAAU;AAChC,UAAI,UAAU,OAAO,WAAW,YAAY,CAAC,MAAM,QAAQ,MAAM,GAAG;AAClE,gBAAQ,KAAK,EAAE,KAAK,QAAQ,GAAG,OAA0C,CAAC;AAAA,MAC5E,OAAO;AACL,eAAO,KAAK,EAAE,KAAK,QAAQ,GAAG,SAAS,gBAAgB,CAAC;AAAA,MAC1D;AAAA,IACF,CAAC;AAAA,EACH,OAAO;AACL,UAAM,CAAC,QAAQ,GAAG,IAAI,IAAI,SAAS,IAAI;AACvC,QAAI,CAAC,UAAU,QAAQ,MAAM,GAAG;AAC9B,aAAO,EAAE,OAAO,CAAC,GAAG,QAAQ,CAAC,EAAE,KAAK,GAAG,SAAS,oBAAoB,CAAC,EAAE;AAAA,IACzE;AACA,SAAK,QAAQ,CAAC,OAAO,UAAU;AAE7B,UAAI,QAAQ,KAAK,EAAG;AACpB,YAAM,SAAiC,CAAC;AACxC,aAAO,QAAQ,CAAC,QAAQ,MAAM;AAC5B,YAAI,OAAO,KAAK,EAAG,QAAO,OAAO,KAAK,CAAC,IAAI,MAAM,CAAC,KAAK;AAAA,MACzD,CAAC;AACD,cAAQ,KAAK,EAAE,KAAK,QAAQ,GAAG,OAAO,CAAC;AAAA,IACzC,CAAC;AAAA,EACH;AAEA,QAAM,QAAuB,CAAC;AAC9B,QAAM,OAAO,oBAAI,IAAY;AAC7B,aAAW,EAAE,KAAK,OAAO,KAAK,SAAS;AACrC,UAAM,SAAS,OAAO,MAAM;AAC5B,QAAI,OAAO,WAAW,UAAU;AAC9B,aAAO,KAAK,EAAE,KAAK,SAAS,OAAO,CAAC;AAAA,IACtC,WAAW,KAAK,IAAI,OAAO,EAAE,GAAG;AAC9B,aAAO,KAAK,EAAE,KAAK,SAAS,iBAAiB,OAAO,EAAE,wBAAwB,CAAC;AAAA,IACjF,OAAO;AACL,WAAK,IAAI,OAAO,EAAE;AAClB,YAAM,KAAK,MAAM;AAAA,IACnB;AAAA,EACF;AACA,SAAO,EAAE,OAAO,OAAO;AACzB;","names":["slug","value"]}
|
package/package.json
CHANGED
|
@@ -1,6 +1,69 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@franyerhiguerey/ai-chat-ingest",
|
|
3
|
-
"version": "0.
|
|
4
|
-
"
|
|
5
|
-
"
|
|
3
|
+
"version": "0.2.0",
|
|
4
|
+
"description": "Turns uploaded files (md, txt, HTML, PDF, DOCX) into ai-chat knowledge documents",
|
|
5
|
+
"keywords": [
|
|
6
|
+
"chatbot",
|
|
7
|
+
"pdf",
|
|
8
|
+
"docx",
|
|
9
|
+
"csv",
|
|
10
|
+
"knowledge-base"
|
|
11
|
+
],
|
|
12
|
+
"license": "MIT",
|
|
13
|
+
"author": "Franyer Higuerey (https://github.com/FranyerHiguerey)",
|
|
14
|
+
"homepage": "https://github.com/FranyerHiguerey/ai-chat/tree/main/packages/ingest#readme",
|
|
15
|
+
"repository": {
|
|
16
|
+
"type": "git",
|
|
17
|
+
"url": "git+https://github.com/FranyerHiguerey/ai-chat.git",
|
|
18
|
+
"directory": "packages/ingest"
|
|
19
|
+
},
|
|
20
|
+
"bugs": {
|
|
21
|
+
"url": "https://github.com/FranyerHiguerey/ai-chat/issues"
|
|
22
|
+
},
|
|
23
|
+
"type": "module",
|
|
24
|
+
"main": "./dist/index.js",
|
|
25
|
+
"types": "./dist/index.d.ts",
|
|
26
|
+
"exports": {
|
|
27
|
+
".": {
|
|
28
|
+
"types": "./dist/index.d.ts",
|
|
29
|
+
"import": "./dist/index.js"
|
|
30
|
+
}
|
|
31
|
+
},
|
|
32
|
+
"files": [
|
|
33
|
+
"dist"
|
|
34
|
+
],
|
|
35
|
+
"publishConfig": {
|
|
36
|
+
"access": "public"
|
|
37
|
+
},
|
|
38
|
+
"sideEffects": false,
|
|
39
|
+
"engines": {
|
|
40
|
+
"node": ">=20"
|
|
41
|
+
},
|
|
42
|
+
"peerDependencies": {
|
|
43
|
+
"mammoth": "*",
|
|
44
|
+
"unpdf": "*"
|
|
45
|
+
},
|
|
46
|
+
"peerDependenciesMeta": {
|
|
47
|
+
"mammoth": {
|
|
48
|
+
"optional": true
|
|
49
|
+
},
|
|
50
|
+
"unpdf": {
|
|
51
|
+
"optional": true
|
|
52
|
+
}
|
|
53
|
+
},
|
|
54
|
+
"devDependencies": {
|
|
55
|
+
"@types/node": "^22.10.0",
|
|
56
|
+
"mammoth": "^1.13.0",
|
|
57
|
+
"tsup": "^8.3.5",
|
|
58
|
+
"typescript": "^5.7.3",
|
|
59
|
+
"unpdf": "^1.9.1",
|
|
60
|
+
"vitest": "^3.0.0"
|
|
61
|
+
},
|
|
62
|
+
"scripts": {
|
|
63
|
+
"build": "tsup",
|
|
64
|
+
"dev": "tsup --watch",
|
|
65
|
+
"lint": "eslint .",
|
|
66
|
+
"typecheck": "tsc --noEmit",
|
|
67
|
+
"test": "vitest run"
|
|
68
|
+
}
|
|
6
69
|
}
|