@xeplr/actions 1.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/index.js +51 -0
- package/lib/builtins/db/fetch.js +142 -0
- package/lib/builtins/db/list-columns.js +39 -0
- package/lib/builtins/db/list-procedures.js +36 -0
- package/lib/builtins/db/list-tables.js +38 -0
- package/lib/builtins/db/list-views.js +35 -0
- package/lib/builtins/db/push.js +107 -0
- package/lib/builtins/db/query.js +5 -0
- package/lib/builtins/file/move.js +8 -0
- package/lib/builtins/file/upload.js +123 -0
- package/lib/builtins/http-request.js +3 -0
- package/lib/builtins/index.js +40 -0
- package/lib/builtins/send-email.js +8 -0
- package/lib/builtins/spawn-program.js +4 -0
- package/lib/drivers/db/index.js +41 -0
- package/lib/drivers/db/mongo.js +8 -0
- package/lib/drivers/db/mssql.js +464 -0
- package/lib/drivers/db/mysql.js +425 -0
- package/lib/drivers/db/postgres.js +372 -0
- package/lib/drivers/file/google.js +8 -0
- package/lib/drivers/file/index.js +42 -0
- package/lib/drivers/file/local.js +68 -0
- package/lib/drivers/file/sftp.js +9 -0
- package/lib/drivers/file/sharepoint.js +9 -0
- package/lib/errors.js +37 -0
- package/lib/formats/_xlsx-stream.js +355 -0
- package/lib/formats/csv.js +54 -0
- package/lib/formats/excel.js +87 -0
- package/lib/formats/index.js +38 -0
- package/lib/formats/json.js +42 -0
- package/lib/formats/txt.js +6 -0
- package/lib/registry.js +75 -0
- package/lib/runner.js +81 -0
- package/lib/streaming/checkpoint.js +46 -0
- package/lib/streaming/index.js +10 -0
- package/lib/streaming/resume.js +35 -0
- package/lib/streaming/spool.js +253 -0
- package/lib/streaming/utils.js +82 -0
- package/lib/uploader/attach.js +69 -0
- package/lib/uploader/index.js +298 -0
- package/lib/uploader/infer.js +76 -0
- package/lib/uploader/meta-store-knex.js +136 -0
- package/lib/uploader/meta-store-noop.js +11 -0
- package/lib/uploader/reconcile.js +88 -0
- package/package.json +90 -0
|
@@ -0,0 +1,355 @@
|
|
|
1
|
+
// _xlsx-stream — constant-memory streaming reader for .xlsx worksheets.
|
|
2
|
+
//
|
|
3
|
+
// Ported from polus/db.helpers/xlsxStreamReader.js. See MEMORY entry
|
|
4
|
+
// [[polus-xlsxstreamreader]] for the origin.
|
|
5
|
+
//
|
|
6
|
+
// WHY THIS EXISTS: the `xlsx-stream-reader` npm package matches on the FULL
|
|
7
|
+
// tag name (`<row>`, `<c>`). Files authored by the .NET OpenXML SDK — which
|
|
8
|
+
// includes SharePoint downloads and Microsoft Graph exports — emit the SAME
|
|
9
|
+
// OOXML schema but with a namespace PREFIX (`<x:row>`, `<x:c>`). That reader
|
|
10
|
+
// then matches nothing, walks the WHOLE worksheet XML looking for tags that
|
|
11
|
+
// never appear, and burns unbounded heap. This reader matches on the LOCAL
|
|
12
|
+
// name (prefix stripped), so it reads both.
|
|
13
|
+
//
|
|
14
|
+
// It streams the worksheet part through `unzipper` (the 1.3GB-XML case is
|
|
15
|
+
// entirely disk-bound; peak RSS on a 84MB SharePoint xlsx is ~130MB) and
|
|
16
|
+
// SAX-parses rows with `saxes`. sharedStrings is loaded once (streamed too)
|
|
17
|
+
// only when present; inline-string files skip that entirely.
|
|
18
|
+
//
|
|
19
|
+
// Interface:
|
|
20
|
+
//
|
|
21
|
+
// stream_xlsx_sheet_rows(filePath, sheet_name, { onHeaders, onRow })
|
|
22
|
+
// -> Promise<{ sheet_name, headers, row_count, part, date_columns }>
|
|
23
|
+
//
|
|
24
|
+
// Dates arrive in each onRow record as JS Date objects (Excel has no date
|
|
25
|
+
// cell type; a date is a numeric cell whose style resolves to a date
|
|
26
|
+
// number-format via styles.xml).
|
|
27
|
+
|
|
28
|
+
var unzipper = require('unzipper');
|
|
29
|
+
var { SaxesParser } = require('saxes');
|
|
30
|
+
var { StringDecoder } = require('string_decoder');
|
|
31
|
+
|
|
32
|
+
// Strip an XML namespace prefix: "x:row" -> "row", "row" -> "row".
|
|
33
|
+
function localName(n) {
|
|
34
|
+
if (!n) return n;
|
|
35
|
+
var i = n.indexOf(':');
|
|
36
|
+
return i === -1 ? n : n.slice(i + 1);
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
// Read an attribute by its LOCAL name (handles prefixed attrs like r:id).
|
|
40
|
+
function attr(attributes, key) {
|
|
41
|
+
if (!attributes) return undefined;
|
|
42
|
+
var keys = Object.keys(attributes);
|
|
43
|
+
for (var j = 0; j < keys.length; j++) {
|
|
44
|
+
if (localName(keys[j]) === key) {
|
|
45
|
+
var v = attributes[keys[j]];
|
|
46
|
+
return (v && typeof v === 'object' && 'value' in v) ? v.value : v;
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
return undefined;
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
// ── Date detection ──────────────────────────────────────────────────────
|
|
53
|
+
// xlsx has NO date cell type. A date is a NUMBER whose cell style
|
|
54
|
+
// (`<c s="..">` indexing into styles.xml's <cellXfs>) carries a date
|
|
55
|
+
// number-format. We resolve style index → numFmtId → is-date.
|
|
56
|
+
|
|
57
|
+
// Built-in numFmtIds that are always date/time formats (ECMA-376 §18.8.30).
|
|
58
|
+
var BUILTIN_DATE_NUMFMT_IDS = new Set([14,15,16,17,18,19,20,21,22,45,46,47]);
|
|
59
|
+
|
|
60
|
+
// Does a (custom) format code describe a date/time? Strip escaped chars,
|
|
61
|
+
// quoted literals, and [bracketed] sections, then look for y/m/d/h/s tokens.
|
|
62
|
+
function isDateFormatCode(code) {
|
|
63
|
+
if (!code) return false;
|
|
64
|
+
var stripped = String(code)
|
|
65
|
+
.replace(/\\./g, '')
|
|
66
|
+
.replace(/"[^"]*"/g, '')
|
|
67
|
+
.replace(/\[[^\]]*\]/g, '');
|
|
68
|
+
return /[ymdhs]/i.test(stripped);
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
function isDateNumFmt(id, code) {
|
|
72
|
+
return BUILTIN_DATE_NUMFMT_IDS.has(id) || (id >= 164 && isDateFormatCode(code));
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
// Excel serial number → JS Date (UTC). Day 0 is 1899-12-30. 25569 = days
|
|
76
|
+
// from 1899-12-30 to Unix epoch. 1904 system (Mac-authored files) shifts +1462.
|
|
77
|
+
function excelSerialToDate(serial, date1904) {
|
|
78
|
+
var s = date1904 ? serial + 1462 : serial;
|
|
79
|
+
return new Date(Math.round((s - 25569) * 86400000));
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
// "AB12" → 0-based column index (A=0, AB=27). Ignores trailing row digits.
|
|
83
|
+
function colRefToIndex(ref) {
|
|
84
|
+
var col = 0;
|
|
85
|
+
for (var i = 0; i < ref.length; i++) {
|
|
86
|
+
var ch = ref.charCodeAt(i);
|
|
87
|
+
if (ch >= 65 && ch <= 90) col = col * 26 + (ch - 64); // A-Z
|
|
88
|
+
else if (ch >= 97 && ch <= 122) col = col * 26 + (ch - 96); // a-z
|
|
89
|
+
else break;
|
|
90
|
+
}
|
|
91
|
+
return col - 1;
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
// SAX-parse a whole (small) XML buffer with per-event callbacks.
|
|
95
|
+
function saxParseString(xml, handlers) {
|
|
96
|
+
var parser = new SaxesParser();
|
|
97
|
+
if (handlers.opentag) parser.on('opentag', handlers.opentag);
|
|
98
|
+
if (handlers.text) parser.on('text', handlers.text);
|
|
99
|
+
if (handlers.closetag) parser.on('closetag', handlers.closetag);
|
|
100
|
+
parser.write(xml);
|
|
101
|
+
parser.close();
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
// SAX-parse a STREAMED zip entry, decoding bytes safely across chunk boundaries.
|
|
105
|
+
function saxParseStream(readable, handlers) {
|
|
106
|
+
return new Promise(function(resolve, reject) {
|
|
107
|
+
var parser = new SaxesParser();
|
|
108
|
+
if (handlers.opentag) parser.on('opentag', handlers.opentag);
|
|
109
|
+
if (handlers.text) parser.on('text', handlers.text);
|
|
110
|
+
if (handlers.closetag) parser.on('closetag', handlers.closetag);
|
|
111
|
+
parser.on('error', reject);
|
|
112
|
+
var decoder = new StringDecoder('utf8');
|
|
113
|
+
readable.on('data', function(chunk) {
|
|
114
|
+
try { parser.write(decoder.write(chunk)); } catch (e) { reject(e); }
|
|
115
|
+
});
|
|
116
|
+
readable.on('end', function() {
|
|
117
|
+
try { parser.write(decoder.end()); parser.close(); resolve(); } catch (e) { reject(e); }
|
|
118
|
+
});
|
|
119
|
+
readable.on('error', reject);
|
|
120
|
+
});
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
function findFile(directory, p) { return directory.files.find(function(f) { return f.path === p; }); }
|
|
124
|
+
|
|
125
|
+
async function readEntryText(directory, p) {
|
|
126
|
+
var f = findFile(directory, p);
|
|
127
|
+
if (!f) return null;
|
|
128
|
+
var buf = await f.buffer();
|
|
129
|
+
return buf.toString('utf8');
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
// workbook.xml → [{ name, rId }] in document order.
|
|
133
|
+
function parseWorkbookSheets(xml) {
|
|
134
|
+
if (!xml) return [];
|
|
135
|
+
var sheets = [];
|
|
136
|
+
saxParseString(xml, {
|
|
137
|
+
opentag: function(tag) {
|
|
138
|
+
if (localName(tag.name) === 'sheet') {
|
|
139
|
+
sheets.push({ name: attr(tag.attributes, 'name'), rId: attr(tag.attributes, 'id') });
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
});
|
|
143
|
+
return sheets;
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
// workbook.xml.rels → { rId: target }
|
|
147
|
+
function parseRels(xml) {
|
|
148
|
+
var map = {};
|
|
149
|
+
if (!xml) return map;
|
|
150
|
+
saxParseString(xml, {
|
|
151
|
+
opentag: function(tag) {
|
|
152
|
+
if (localName(tag.name) === 'Relationship') {
|
|
153
|
+
var id = attr(tag.attributes, 'Id');
|
|
154
|
+
var target = attr(tag.attributes, 'Target');
|
|
155
|
+
if (id) map[id] = target;
|
|
156
|
+
}
|
|
157
|
+
}
|
|
158
|
+
});
|
|
159
|
+
return map;
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
// rels Target → absolute zip part path. Usually relative to xl/,
|
|
163
|
+
// e.g. "worksheets/sheet2.xml"; may be absolute ("/xl/worksheets/sheet2.xml").
|
|
164
|
+
function relTargetToPart(target) {
|
|
165
|
+
if (!target) return null;
|
|
166
|
+
if (target.startsWith('/')) return target.slice(1);
|
|
167
|
+
return 'xl/' + target.replace(/^\.\//, '');
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
function norm(s) { return String(s == null ? '' : s).trim().toLowerCase(); }
|
|
171
|
+
|
|
172
|
+
// Resolve requested sheet (by name; else first) → zip part path.
|
|
173
|
+
function resolveTargetPart(sheets, rels, sheet_name, directory) {
|
|
174
|
+
var withPart = sheets.map(function(s) { return Object.assign({}, s, { part: relTargetToPart(rels[s.rId]) }); });
|
|
175
|
+
var chosen = sheet_name
|
|
176
|
+
? withPart.find(function(s) { return norm(s.name) === norm(sheet_name); })
|
|
177
|
+
: withPart[0];
|
|
178
|
+
if (!chosen && sheet_name && withPart.length === 1) chosen = withPart[0];
|
|
179
|
+
if (chosen && chosen.part && findFile(directory, chosen.part)) return chosen;
|
|
180
|
+
|
|
181
|
+
// Last resort: the largest xl/worksheets/sheetN.xml part actually present
|
|
182
|
+
// (handles broken/missing rels — the data is in the big worksheet part).
|
|
183
|
+
var wsFiles = directory.files
|
|
184
|
+
.filter(function(f) { return /xl\/worksheets\/sheet\d+\.xml$/.test(f.path); })
|
|
185
|
+
.sort(function(a, b) { return (b.uncompressedSize || 0) - (a.uncompressedSize || 0); });
|
|
186
|
+
if (wsFiles.length) {
|
|
187
|
+
return { name: sheet_name || (chosen && chosen.name) || '(first sheet)', part: wsFiles[0].path };
|
|
188
|
+
}
|
|
189
|
+
return null;
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
// sharedStrings.xml → [string]. Returns null when file has none (inline strings only).
|
|
193
|
+
async function parseSharedStrings(directory) {
|
|
194
|
+
var f = findFile(directory, 'xl/sharedStrings.xml');
|
|
195
|
+
if (!f) return null;
|
|
196
|
+
var strings = [];
|
|
197
|
+
var cur = null, inT = 0, buf = '';
|
|
198
|
+
await saxParseStream(f.stream(), {
|
|
199
|
+
opentag: function(tag) {
|
|
200
|
+
var ln = localName(tag.name);
|
|
201
|
+
if (ln === 'si') { cur = ''; }
|
|
202
|
+
else if (ln === 't') { inT++; buf = ''; }
|
|
203
|
+
},
|
|
204
|
+
text: function(t) { if (inT > 0) buf += t; },
|
|
205
|
+
closetag: function(tag) {
|
|
206
|
+
var ln = localName(tag.name);
|
|
207
|
+
if (ln === 't') { inT--; cur = (cur || '') + buf; buf = ''; }
|
|
208
|
+
else if (ln === 'si') { strings.push(cur || ''); cur = null; }
|
|
209
|
+
}
|
|
210
|
+
});
|
|
211
|
+
return strings;
|
|
212
|
+
}
|
|
213
|
+
|
|
214
|
+
// styles.xml → boolean[] indexed by cell-style position: is style i a date format?
|
|
215
|
+
async function parseStyles(directory) {
|
|
216
|
+
var xml = await readEntryText(directory, 'xl/styles.xml');
|
|
217
|
+
if (!xml) return [];
|
|
218
|
+
|
|
219
|
+
var numFmtCode = {}; // custom numFmtId → formatCode
|
|
220
|
+
var cellXfNumFmtIds = []; // cellXfs xf index → numFmtId
|
|
221
|
+
var inCellXfs = false;
|
|
222
|
+
|
|
223
|
+
saxParseString(xml, {
|
|
224
|
+
opentag: function(tag) {
|
|
225
|
+
var ln = localName(tag.name);
|
|
226
|
+
if (ln === 'numFmt') {
|
|
227
|
+
var id = parseInt(attr(tag.attributes, 'numFmtId'), 10);
|
|
228
|
+
if (!Number.isNaN(id)) numFmtCode[id] = attr(tag.attributes, 'formatCode');
|
|
229
|
+
} else if (ln === 'cellXfs') {
|
|
230
|
+
inCellXfs = true;
|
|
231
|
+
} else if (ln === 'xf' && inCellXfs) {
|
|
232
|
+
var xid = parseInt(attr(tag.attributes, 'numFmtId') || '0', 10);
|
|
233
|
+
cellXfNumFmtIds.push(Number.isNaN(xid) ? 0 : xid);
|
|
234
|
+
}
|
|
235
|
+
},
|
|
236
|
+
closetag: function(tag) {
|
|
237
|
+
if (localName(tag.name) === 'cellXfs') inCellXfs = false;
|
|
238
|
+
}
|
|
239
|
+
});
|
|
240
|
+
|
|
241
|
+
return cellXfNumFmtIds.map(function(id) { return isDateNumFmt(id, numFmtCode[id]); });
|
|
242
|
+
}
|
|
243
|
+
|
|
244
|
+
// List sheet names in document order — for a "pick a sheet" UI step, before
|
|
245
|
+
// committing to streaming any one sheet's rows. Cheap: only reads
|
|
246
|
+
// workbook.xml, not any worksheet part.
|
|
247
|
+
async function list_xlsx_sheets(filePath) {
|
|
248
|
+
var directory = await unzipper.Open.file(filePath);
|
|
249
|
+
var wbXml = await readEntryText(directory, 'xl/workbook.xml');
|
|
250
|
+
return parseWorkbookSheets(wbXml).map(function(s) { return s.name; });
|
|
251
|
+
}
|
|
252
|
+
|
|
253
|
+
// ── main API ─────────────────────────────────────────────────────────────
|
|
254
|
+
async function stream_xlsx_sheet_rows(filePath, sheet_name, handlers) {
|
|
255
|
+
handlers = handlers || {};
|
|
256
|
+
var onHeaders = handlers.onHeaders;
|
|
257
|
+
var onRow = handlers.onRow;
|
|
258
|
+
var logFn = handlers.logFunction || function() {};
|
|
259
|
+
|
|
260
|
+
var directory = await unzipper.Open.file(filePath);
|
|
261
|
+
|
|
262
|
+
var wbXml = await readEntryText(directory, 'xl/workbook.xml');
|
|
263
|
+
var relsXml = await readEntryText(directory, 'xl/_rels/workbook.xml.rels');
|
|
264
|
+
var target = resolveTargetPart(parseWorkbookSheets(wbXml), parseRels(relsXml), sheet_name, directory);
|
|
265
|
+
if (!target || !target.part || !findFile(directory, target.part)) {
|
|
266
|
+
throw new Error('xlsx-stream: could not resolve worksheet part for sheet "' + (sheet_name || '(first)') + '"');
|
|
267
|
+
}
|
|
268
|
+
logFn('xlsx-stream: reading "' + (target.name || sheet_name) + '" from ' + target.part);
|
|
269
|
+
|
|
270
|
+
var sharedStrings = await parseSharedStrings(directory);
|
|
271
|
+
var dateStyle = await parseStyles(directory);
|
|
272
|
+
var date1904 = /date1904\s*=\s*"?(1|true)"?/i.test(wbXml || '');
|
|
273
|
+
|
|
274
|
+
var headers = null;
|
|
275
|
+
var headerByCol = {};
|
|
276
|
+
var row_count = 0;
|
|
277
|
+
var dateColIdx = new Set();
|
|
278
|
+
|
|
279
|
+
// per-cell SAX state
|
|
280
|
+
var rowCells = {};
|
|
281
|
+
var curCol = -1, curType = null, curStyle = 0, capturing = false, textBuf = '';
|
|
282
|
+
|
|
283
|
+
function commitCellText(closingLocal) {
|
|
284
|
+
var val;
|
|
285
|
+
if (curType === 's') {
|
|
286
|
+
var idx = parseInt(textBuf, 10);
|
|
287
|
+
val = (sharedStrings && sharedStrings[idx] != null) ? sharedStrings[idx] : null;
|
|
288
|
+
} else if (curType === 'b') {
|
|
289
|
+
val = textBuf === '1';
|
|
290
|
+
} else if (curType === 'str' || curType === 'inlineStr' || closingLocal === 't') {
|
|
291
|
+
val = textBuf;
|
|
292
|
+
} else {
|
|
293
|
+
var n = Number(textBuf);
|
|
294
|
+
val = (textBuf !== '' && !Number.isNaN(n)) ? n : (textBuf === '' ? null : textBuf);
|
|
295
|
+
if (typeof val === 'number' && dateStyle[curStyle]) {
|
|
296
|
+
val = excelSerialToDate(val, date1904);
|
|
297
|
+
if (curCol >= 0) dateColIdx.add(curCol);
|
|
298
|
+
}
|
|
299
|
+
}
|
|
300
|
+
if (curCol >= 0) rowCells[curCol] = val;
|
|
301
|
+
}
|
|
302
|
+
|
|
303
|
+
await saxParseStream(findFile(directory, target.part).stream(), {
|
|
304
|
+
opentag: function(tag) {
|
|
305
|
+
var ln = localName(tag.name);
|
|
306
|
+
if (ln === 'row') { rowCells = {}; }
|
|
307
|
+
else if (ln === 'c') {
|
|
308
|
+
var ref = attr(tag.attributes, 'r');
|
|
309
|
+
curCol = ref ? colRefToIndex(ref) : (curCol + 1);
|
|
310
|
+
curType = attr(tag.attributes, 't') || null;
|
|
311
|
+
var s = attr(tag.attributes, 's');
|
|
312
|
+
curStyle = s != null ? parseInt(s, 10) || 0 : 0;
|
|
313
|
+
}
|
|
314
|
+
else if (ln === 'v' || ln === 't') { capturing = true; textBuf = ''; }
|
|
315
|
+
},
|
|
316
|
+
text: function(t) { if (capturing) textBuf += t; },
|
|
317
|
+
closetag: function(tag) {
|
|
318
|
+
var ln = localName(tag.name);
|
|
319
|
+
if (ln === 'v' || ln === 't') { commitCellText(ln); capturing = false; textBuf = ''; }
|
|
320
|
+
else if (ln === 'row') {
|
|
321
|
+
if (!headers) {
|
|
322
|
+
headers = [];
|
|
323
|
+
var cols = Object.keys(rowCells).map(Number);
|
|
324
|
+
var maxCol = cols.length ? Math.max.apply(null, cols) : -1;
|
|
325
|
+
for (var i = 0; i <= maxCol; i++) {
|
|
326
|
+
var h = String(rowCells[i] == null ? '' : rowCells[i]).trim();
|
|
327
|
+
headers.push(h);
|
|
328
|
+
headerByCol[i] = h;
|
|
329
|
+
}
|
|
330
|
+
if (typeof onHeaders === 'function') onHeaders(headers);
|
|
331
|
+
} else {
|
|
332
|
+
var rec = {};
|
|
333
|
+
for (var j = 0; j < headers.length; j++) {
|
|
334
|
+
var hk = headerByCol[j];
|
|
335
|
+
if (hk) rec[hk] = (rowCells[j] === undefined ? null : rowCells[j]);
|
|
336
|
+
}
|
|
337
|
+
if (typeof onRow === 'function') onRow(rec);
|
|
338
|
+
row_count++;
|
|
339
|
+
}
|
|
340
|
+
}
|
|
341
|
+
}
|
|
342
|
+
});
|
|
343
|
+
|
|
344
|
+
var date_columns = Array.from(dateColIdx).map(function(i) { return headerByCol[i]; }).filter(Boolean);
|
|
345
|
+
|
|
346
|
+
return {
|
|
347
|
+
sheet_name: sheet_name || target.name || '(first sheet)',
|
|
348
|
+
headers: headers || [],
|
|
349
|
+
row_count: row_count,
|
|
350
|
+
part: target.part,
|
|
351
|
+
date_columns: date_columns
|
|
352
|
+
};
|
|
353
|
+
}
|
|
354
|
+
|
|
355
|
+
module.exports = { stream_xlsx_sheet_rows: stream_xlsx_sheet_rows, list_xlsx_sheets: list_xlsx_sheets };
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
// CSV parser — accepts a readable stream of CSV data, yields row objects.
|
|
2
|
+
// Peer dep: csv-parse (declared as optional in @xeplr/actions/package.json).
|
|
3
|
+
//
|
|
4
|
+
// parseStream(stream, opts?) → async iterable<row>
|
|
5
|
+
// opts.columns true (default) uses the first row as headers.
|
|
6
|
+
// Pass an array to override header names.
|
|
7
|
+
// opts.delimiter default ','
|
|
8
|
+
// opts.cast default true — csv-parse infers numbers/booleans
|
|
9
|
+
// from strings automatically. Turn off if you want raw strings.
|
|
10
|
+
// opts.skipEmptyLines default true
|
|
11
|
+
// opts.trim default true
|
|
12
|
+
|
|
13
|
+
module.exports = {
|
|
14
|
+
requires: ['csv-parse'],
|
|
15
|
+
|
|
16
|
+
async *parseStream(stream, opts) {
|
|
17
|
+
opts = opts || {};
|
|
18
|
+
var { parse } = require('csv-parse');
|
|
19
|
+
|
|
20
|
+
// Custom cast: empty → null, 'true'/'false' → boolean, numeric → number,
|
|
21
|
+
// everything else → string. csv-parse's default `cast: true` handles only
|
|
22
|
+
// numbers, so we roll our own to also pick up booleans (common in CSVs).
|
|
23
|
+
function autoCast(value, context) {
|
|
24
|
+
if (context && context.header) return value;
|
|
25
|
+
if (value === '' || value == null) return null;
|
|
26
|
+
var lower = value.toLowerCase();
|
|
27
|
+
if (lower === 'true') return true;
|
|
28
|
+
if (lower === 'false') return false;
|
|
29
|
+
if (/^-?\d+(\.\d+)?$/.test(value)) return Number(value);
|
|
30
|
+
return value;
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
var parserOpts = Object.assign({
|
|
34
|
+
columns: opts.columns !== undefined ? opts.columns : true,
|
|
35
|
+
delimiter: opts.delimiter || ',',
|
|
36
|
+
cast: opts.cast !== undefined ? opts.cast : autoCast,
|
|
37
|
+
cast_date: opts.cast_date !== undefined ? opts.cast_date : false,
|
|
38
|
+
skip_empty_lines: opts.skipEmptyLines !== undefined ? opts.skipEmptyLines : true,
|
|
39
|
+
trim: opts.trim !== undefined ? opts.trim : true,
|
|
40
|
+
// Strip UTF-8 BOM from stream start. Windows Excel exports CSVs
|
|
41
|
+
// with a BOM () and it silently corrupts the first header
|
|
42
|
+
// name if not removed. Zero downside — the only edge case is a
|
|
43
|
+
// field whose first byte is literally , which is essentially
|
|
44
|
+
// never a real file.
|
|
45
|
+
bom: opts.bom !== undefined ? opts.bom : true
|
|
46
|
+
}, opts.forwardOpts || {});
|
|
47
|
+
|
|
48
|
+
var parser = stream.pipe(parse(parserOpts));
|
|
49
|
+
|
|
50
|
+
for await (var row of parser) {
|
|
51
|
+
yield row;
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
};
|
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
// Excel parser — streams .xlsx via the polus-derived reader (unzipper + saxes).
|
|
2
|
+
//
|
|
3
|
+
// See lib/formats/_xlsx-stream.js for the WHY (SharePoint / .NET OpenXML
|
|
4
|
+
// files use namespace-prefixed tags — <x:row>, <x:c> — which the popular
|
|
5
|
+
// `xlsx-stream-reader` npm package silently fails to parse, burning heap
|
|
6
|
+
// while emitting zero rows).
|
|
7
|
+
//
|
|
8
|
+
// Peer deps: `unzipper`, `saxes` — both small, both truly streaming.
|
|
9
|
+
//
|
|
10
|
+
// Interface:
|
|
11
|
+
// inputMode: 'path' — needs a file path (unzipper seeks)
|
|
12
|
+
// parsePath(filePath, opts?) → async iterable<row>
|
|
13
|
+
//
|
|
14
|
+
// opts.sheet — sheet name to read (default: first / largest sheet)
|
|
15
|
+
// opts.startRow — skip N leading rows AFTER headers (default 0)
|
|
16
|
+
// opts.columns — true (default): first row is headers, yield objects.
|
|
17
|
+
// false: yield arrays. Not supported yet — polus reader
|
|
18
|
+
// always uses header names.
|
|
19
|
+
// Values arrive TYPED from Excel cell metadata:
|
|
20
|
+
// date-styled numbers → Date objects (identify with `value instanceof Date`)
|
|
21
|
+
// number cells → numbers
|
|
22
|
+
// boolean cells → booleans
|
|
23
|
+
// string cells → strings (from sharedStrings or inline)
|
|
24
|
+
//
|
|
25
|
+
// Memory / backpressure: `saxes` parses synchronously per data chunk and
|
|
26
|
+
// doesn't await our onRow callback, so we can't truly pause the SAX parser
|
|
27
|
+
// from userspace. Instead we buffer rows in an unbounded queue and yield
|
|
28
|
+
// them out; peak memory is bounded by how fast the downstream consumer
|
|
29
|
+
// pulls from the iterator. The uploader routes rows through `spool()` which
|
|
30
|
+
// flushes each batch to a rotating NDJSON file — that pulls the queue empty
|
|
31
|
+
// on a fast, disk-bound cadence and keeps peak RSS in the hundreds-of-MB
|
|
32
|
+
// range even on 1.3GB sheet.xml streams.
|
|
33
|
+
|
|
34
|
+
var { stream_xlsx_sheet_rows, list_xlsx_sheets } = require('./_xlsx-stream');
|
|
35
|
+
|
|
36
|
+
module.exports = {
|
|
37
|
+
requires: ['unzipper', 'saxes'],
|
|
38
|
+
inputMode: 'path',
|
|
39
|
+
|
|
40
|
+
// Sheet names in document order — for a "pick a sheet" UI step before
|
|
41
|
+
// parsing any one sheet's rows via parsePath(filePath, {sheet}).
|
|
42
|
+
async listSheets(filePath) {
|
|
43
|
+
return list_xlsx_sheets(filePath);
|
|
44
|
+
},
|
|
45
|
+
|
|
46
|
+
async *parsePath(filePath, opts) {
|
|
47
|
+
opts = opts || {};
|
|
48
|
+
if (opts.columns === false) {
|
|
49
|
+
throw new Error('excel: columns:false (array rows) is not supported by the streaming reader. Omit or set true.');
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
var startRow = opts.startRow || 0;
|
|
53
|
+
var queue = [];
|
|
54
|
+
var done = false;
|
|
55
|
+
var err = null;
|
|
56
|
+
var skipped = 0;
|
|
57
|
+
|
|
58
|
+
var consumerWait = null;
|
|
59
|
+
function wakeConsumer() { if (consumerWait) { consumerWait(); consumerWait = null; } }
|
|
60
|
+
|
|
61
|
+
var readerDone = stream_xlsx_sheet_rows(filePath, opts.sheet || null, {
|
|
62
|
+
onRow: function(rec) {
|
|
63
|
+
if (skipped < startRow) { skipped++; return; }
|
|
64
|
+
queue.push(rec);
|
|
65
|
+
wakeConsumer();
|
|
66
|
+
}
|
|
67
|
+
}).then(function() {
|
|
68
|
+
done = true;
|
|
69
|
+
wakeConsumer();
|
|
70
|
+
}, function(e) {
|
|
71
|
+
err = e;
|
|
72
|
+
done = true;
|
|
73
|
+
wakeConsumer();
|
|
74
|
+
});
|
|
75
|
+
|
|
76
|
+
try {
|
|
77
|
+
while (true) {
|
|
78
|
+
if (err) throw err;
|
|
79
|
+
if (queue.length) { yield queue.shift(); continue; }
|
|
80
|
+
if (done) return;
|
|
81
|
+
await new Promise(function(resolve) { consumerWait = resolve; });
|
|
82
|
+
}
|
|
83
|
+
} finally {
|
|
84
|
+
try { await readerDone; } catch (_) { /* already surfaced via err */ }
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
};
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
// File format factory. Each action calls getFormat(type) to get the right
|
|
2
|
+
// parser, then invokes the common parser interface.
|
|
3
|
+
//
|
|
4
|
+
// Adding a new format = add a file here + register it below.
|
|
5
|
+
|
|
6
|
+
var FORMATS = {
|
|
7
|
+
excel: function() { return require('./excel'); },
|
|
8
|
+
csv: function() { return require('./csv'); },
|
|
9
|
+
txt: function() { return require('./txt'); },
|
|
10
|
+
json: function() { return require('./json'); }
|
|
11
|
+
};
|
|
12
|
+
|
|
13
|
+
function getFormat(type) {
|
|
14
|
+
var loader = FORMATS[type];
|
|
15
|
+
if (!loader) {
|
|
16
|
+
throw new Error('Unknown format: "' + type + '". Supported: ' + Object.keys(FORMATS).join(', '));
|
|
17
|
+
}
|
|
18
|
+
return loader();
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
function checkFormatRequires(actionName, format, mod) {
|
|
22
|
+
var requires = (mod && mod.requires) || [];
|
|
23
|
+
for (var i = 0; i < requires.length; i++) {
|
|
24
|
+
try { require.resolve(requires[i]); }
|
|
25
|
+
catch (_) {
|
|
26
|
+
throw new Error(
|
|
27
|
+
actionName + ' requires "' + requires[i] + '" for format="' + format + '". ' +
|
|
28
|
+
'Install with: npm install ' + requires[i]
|
|
29
|
+
);
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
module.exports = {
|
|
35
|
+
getFormat: getFormat,
|
|
36
|
+
checkFormatRequires: checkFormatRequires,
|
|
37
|
+
SUPPORTED: Object.keys(FORMATS)
|
|
38
|
+
};
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
// JSON parser — buffers the whole stream (no streaming-JSON dependency —
|
|
2
|
+
// CSV/Excel already handle the genuinely-large-file case via true streaming;
|
|
3
|
+
// JSON uploads are typically smaller structured data, so buffering the whole
|
|
4
|
+
// file is an acceptable simplification here). Accepts either shape,
|
|
5
|
+
// auto-detected:
|
|
6
|
+
// - a single JSON array of row objects: [{...}, {...}]
|
|
7
|
+
// - NDJSON: one JSON object per line
|
|
8
|
+
//
|
|
9
|
+
// parseStream(stream, opts?) → async iterable<row>
|
|
10
|
+
|
|
11
|
+
module.exports = {
|
|
12
|
+
requires: [],
|
|
13
|
+
|
|
14
|
+
async *parseStream(stream, opts) {
|
|
15
|
+
opts = opts || {};
|
|
16
|
+
var chunks = [];
|
|
17
|
+
for await (var chunk of stream) {
|
|
18
|
+
chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk));
|
|
19
|
+
}
|
|
20
|
+
var text = Buffer.concat(chunks).toString('utf8').trim();
|
|
21
|
+
if (!text) return;
|
|
22
|
+
|
|
23
|
+
// A single JSON document (array or object) parses cleanly as one value;
|
|
24
|
+
// NDJSON does not (multiple top-level values) — JSON.parse failing is
|
|
25
|
+
// the signal to fall back to line-by-line.
|
|
26
|
+
try {
|
|
27
|
+
var parsed = JSON.parse(text);
|
|
28
|
+
var rows = Array.isArray(parsed) ? parsed : [parsed];
|
|
29
|
+
for (var i = 0; i < rows.length; i++) yield rows[i];
|
|
30
|
+
return;
|
|
31
|
+
} catch (_) {
|
|
32
|
+
// fall through to NDJSON
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
var lines = text.split('\n');
|
|
36
|
+
for (var j = 0; j < lines.length; j++) {
|
|
37
|
+
var line = lines[j].trim();
|
|
38
|
+
if (!line) continue;
|
|
39
|
+
yield JSON.parse(line);
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
};
|
package/lib/registry.js
ADDED
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
// In-memory registry of actions. Consuming apps call register(...) at
|
|
2
|
+
// startup; the runner and any UI look actions up by name at run time.
|
|
3
|
+
//
|
|
4
|
+
// Definition shape:
|
|
5
|
+
// {
|
|
6
|
+
// name : string — required (unless passed as first arg)
|
|
7
|
+
// description : string — human-readable
|
|
8
|
+
// inputSchema : array — [{name,type,required,default,description,order}]
|
|
9
|
+
// outputSchema : array? — optional; documents what the action returns
|
|
10
|
+
// requires : string[]? — peer-dep module names checked at register time
|
|
11
|
+
// execute : function — async ({ input, system }) => output
|
|
12
|
+
// }
|
|
13
|
+
//
|
|
14
|
+
// Two call forms:
|
|
15
|
+
// register(name, def) → traditional
|
|
16
|
+
// register(def) → module-export style: `def.name` is used
|
|
17
|
+
|
|
18
|
+
var { ActionMissingDependencyError } = require('./errors');
|
|
19
|
+
|
|
20
|
+
var _actions = new Map();
|
|
21
|
+
|
|
22
|
+
function register(nameOrDef, def) {
|
|
23
|
+
var action;
|
|
24
|
+
if (typeof nameOrDef === 'object' && nameOrDef) {
|
|
25
|
+
action = nameOrDef;
|
|
26
|
+
} else {
|
|
27
|
+
action = Object.assign({ name: nameOrDef }, def || {});
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
if (typeof action.name !== 'string' || !action.name) {
|
|
31
|
+
throw new Error('register: action must have a non-empty `name`');
|
|
32
|
+
}
|
|
33
|
+
if (typeof action.execute !== 'function') {
|
|
34
|
+
throw new Error('register: action "' + action.name + '" must have an `execute` function');
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
// Sanity-check peer deps.
|
|
38
|
+
if (Array.isArray(action.requires)) {
|
|
39
|
+
for (var i = 0; i < action.requires.length; i++) {
|
|
40
|
+
var dep = action.requires[i];
|
|
41
|
+
try { require.resolve(dep); }
|
|
42
|
+
catch (_) { throw new ActionMissingDependencyError(action.name, dep); }
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
_actions.set(action.name, {
|
|
47
|
+
description: action.description || '',
|
|
48
|
+
inputSchema: Array.isArray(action.inputSchema) ? action.inputSchema : [],
|
|
49
|
+
outputSchema: action.outputSchema || null,
|
|
50
|
+
requires: Array.isArray(action.requires) ? action.requires.slice() : [],
|
|
51
|
+
execute: action.execute
|
|
52
|
+
});
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
function get(name) { return _actions.get(name); }
|
|
56
|
+
function has(name) { return _actions.has(name); }
|
|
57
|
+
|
|
58
|
+
// Serializable list — suitable for /actions HTTP endpoint response.
|
|
59
|
+
function list() {
|
|
60
|
+
var out = [];
|
|
61
|
+
_actions.forEach(function(def, name) {
|
|
62
|
+
out.push({
|
|
63
|
+
name: name,
|
|
64
|
+
description: def.description,
|
|
65
|
+
inputSchema: def.inputSchema,
|
|
66
|
+
outputSchema: def.outputSchema,
|
|
67
|
+
requires: def.requires
|
|
68
|
+
});
|
|
69
|
+
});
|
|
70
|
+
return out;
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
function clear() { _actions.clear(); }
|
|
74
|
+
|
|
75
|
+
module.exports = { register: register, get: get, has: has, list: list, clear: clear };
|