@idea1/filekit 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (3) hide show
  1. package/README.md +72 -0
  2. package/filekit.js +498 -0
  3. package/package.json +25 -0
package/README.md ADDED
@@ -0,0 +1,72 @@
1
+ # @datafabriq/filekit
2
+
3
+ A reader and writer for the spreadsheets agent-driven data pipelines depend on — CSV and
4
+ XLSX files a client uploads that a downstream process has to parse correctly every time.
5
+ Pure Node (`fs` + `zlib`), no dependencies, no build step.
6
+
7
+ ```js
8
+ const fk = require('@datafabriq/filekit');
9
+
10
+ fk.parseCsv(text, { headerRow }) // quoted fields, comma-thousands; headerRow skips a preamble
11
+ fk.readXlsx(pathOrBuffer) // reads .xlsx directly — the unzip is built in
12
+ fk.writeXlsx(path, sheets) // one tab per dataset; row 0 is the header
13
+
14
+ fk.num(v) // '$1,234.50' → 1234.5; '(312,571.01)' → -312571.01
15
+ fk.excelSerialToYM(n) // 46143 → '2026-05'
16
+ fk.findMonthColumns(header) // discover a MON YYYY matrix by header, never by index
17
+ fk.findColumnBySerialMonth(sheet, headerRow, '2026-05')
18
+ fk.findRow(sheet, (row, n) => …) // locate a block by its label
19
+ ```
20
+
21
+ `readXlsx` returns real sheet names in workbook order; address a sheet by name
22
+ (`wb.sheet('GL 24500 Accrued Bonus')`) rather than by index. Cells are 1-based and
23
+ Excel-shaped: `sheet.at(12, 3)` is cell `C12`.
24
+
25
+ ## Reading messy source files
26
+
27
+ Two properties of these files break naive parsers, and both are covered by the tests:
28
+
29
+ - **Month columns move.** A commission ledger's `MON YYYY` matrix grows a column each
30
+ period, and a bonus workbook's month headers are Excel serial dates. Locate the close
31
+ month by matching the header; a fixed column index silently reads the wrong month.
32
+ - **A cell's value is not always its text.** Totals are formula cells whose value lives in
33
+ `<v>`; empty cells are self-closing and carry no value at all; text may be a shared
34
+ string or an inline string. `filekit` resolves all four.
35
+
36
+ Locate a block by the labels around it (`Beginning`, `New Accruals`, `Payments`, `Ending`,
37
+ the GL numbers, etc.) rather than by row number, so an inserted row does not shift the read.
38
+
39
+ ## Writing a reference workbook
40
+
41
+ `writeXlsx` produces workbooks a Custom Files connector can ingest — one tab per dataset,
42
+ `snake_case` headers on row 1, and conventions bronze expects:
43
+
44
+ - Dates as literal `YYYY-MM-DD` text, never Excel serials, so `TRY_CAST(… AS DATE)` holds.
45
+ - An omitted cell for a null; an empty cell ingests as `NULL`, `0` does not.
46
+ - Values written as numbers, text as inline strings, XML-escaped.
47
+ - Every column ingests as `STRING`; the reading spec does the `TRY_CAST`.
48
+
49
+ ## Tests
50
+
51
+ ```bash
52
+ npm test
53
+ ```
54
+
55
+ Self-contained — every fixture is built in-process, so the suite carries no customer data
56
+ and runs anywhere. It asserts the cell shapes above: formula cells, self-closing cells,
57
+ shared strings with entities, serial dates, a header below a nine-row preamble, and a
58
+ write→read round-trip preserving nulls, negatives, and escaping.
59
+
60
+ ## Origin
61
+
62
+ Extracted from Concept3D's month-end close kit (`datafabriq-ops` → `context/revenue-reporting/close-tools/`),
63
+ where the same reader already served two teams' file shapes (Concept3D's QuotaPath/bonus
64
+ files, Intake's Amazon/Ed's-workbook files) before this package existed.
65
+
66
+ ## Publishing
67
+
68
+ Published as `@idea1/filekit` (public), same npm org as `@idea1/cli`. Pushing a
69
+ `filekit-v*.*.*` tag runs `.github/workflows/publish-filekit.yml`, which runs `npm run build`
70
+ (the test suite) then `npm run deploy` (`npm publish --access public`), authenticated via
71
+ the `NPM_TOKEN` repo secret — an npm Automation token for a member of the `idea1` npm org.
72
+ Bump the version in `package.json` first, since npm rejects re-publishing an existing version.
package/filekit.js ADDED
@@ -0,0 +1,498 @@
1
+ 'use strict';
2
+ /*
3
+ * filekit — messy-file reader toolkit for the close kit.
4
+ *
5
+ * Reads the uploaded spreadsheets a month-end close depends on: CSVs with a
6
+ * preamble before the real header, and .xlsx workbooks (read directly — the
7
+ * unzip is built in, no npm dependency, no pre-unzipping).
8
+ *
9
+ * Pure Node (fs + zlib). No external packages. Same reader serves every team:
10
+ * Concept3D — QuotaPath commission CSV (MON YYYY matrix), accrued-bonus XLSX.
11
+ * Intake — Amazon Main report CSV (header on row 10), Ed's XLSX.
12
+ *
13
+ * Exports: parseCsv, readXlsx, num, excelSerialToDate, excelSerialToYM,
14
+ * findMonthColumns, findRow, unzip.
15
+ */
16
+
17
+ const fs = require('fs');
18
+ const zlib = require('zlib');
19
+
20
+ /* ------------------------------------------------------------------ *
21
+ * Numbers & dates
22
+ * ------------------------------------------------------------------ */
23
+
24
+ /** Coerce a spreadsheet cell to a number, tolerating $ , % whitespace and (parens) negatives. */
25
+ function num(v) {
26
+ if (v == null || v === '') return 0;
27
+ let s = String(v).trim();
28
+ let neg = false;
29
+ if (/^\(.*\)$/.test(s)) { neg = true; s = s.slice(1, -1); }
30
+ s = s.replace(/[$,%\s]/g, '');
31
+ const x = parseFloat(s);
32
+ if (isNaN(x)) return 0;
33
+ return neg ? -x : x;
34
+ }
35
+
36
+ /** Excel serial date (1900 system) → JS Date (UTC). */
37
+ function excelSerialToDate(n) {
38
+ return new Date(Date.UTC(1899, 11, 30) + Math.round(Number(n)) * 86400000);
39
+ }
40
+
41
+ /** Excel serial date → "YYYY-MM". */
42
+ function excelSerialToYM(n) {
43
+ const d = excelSerialToDate(n);
44
+ return d.getUTCFullYear() + '-' + String(d.getUTCMonth() + 1).padStart(2, '0');
45
+ }
46
+
47
+ /* ------------------------------------------------------------------ *
48
+ * CSV
49
+ * ------------------------------------------------------------------ */
50
+
51
+ /** RFC-4180-ish tokenizer: handles quoted fields, "" escapes, CRLF. → string[][] */
52
+ function tokenizeCsv(text) {
53
+ const rows = [];
54
+ let row = [], field = '', inQ = false;
55
+ for (let i = 0; i < text.length; i++) {
56
+ const ch = text[i];
57
+ if (inQ) {
58
+ if (ch === '"') { if (text[i + 1] === '"') { field += '"'; i++; } else inQ = false; }
59
+ else field += ch;
60
+ } else {
61
+ if (ch === '"') inQ = true;
62
+ else if (ch === ',') { row.push(field); field = ''; }
63
+ else if (ch === '\n') { row.push(field); rows.push(row); row = []; field = ''; }
64
+ else if (ch === '\r') { /* skip */ }
65
+ else field += ch;
66
+ }
67
+ }
68
+ if (field.length || row.length) { row.push(field); rows.push(row); }
69
+ return rows;
70
+ }
71
+
72
+ /**
73
+ * Parse CSV text into header + row objects.
74
+ * @param {string} text
75
+ * @param {object} [opts]
76
+ * @param {number} [opts.headerRow=0] 0-based index of the header line. Set to skip a
77
+ * preamble (e.g. 9 for Amazon's row-10 header).
78
+ * @returns {{header:string[], rows:Array<Object>, raw:string[][]}}
79
+ * Each row is keyed by header name; `row._cells` keeps positional access.
80
+ */
81
+ function parseCsv(text, opts = {}) {
82
+ const headerRow = opts.headerRow ?? 0;
83
+ const raw = tokenizeCsv(text);
84
+ const header = (raw[headerRow] || []).map(h => h.trim());
85
+ const rows = [];
86
+ for (let r = headerRow + 1; r < raw.length; r++) {
87
+ const cells = raw[r];
88
+ if (!cells || (cells.length === 1 && cells[0] === '')) continue; // blank line
89
+ const o = { _cells: cells };
90
+ for (let c = 0; c < header.length; c++) o[header[c]] = cells[c];
91
+ rows.push(o);
92
+ }
93
+ return { header, rows, raw };
94
+ }
95
+
96
+ /**
97
+ * Find the `MON YYYY` matrix columns in a CSV header (e.g. QuotaPath's amortization
98
+ * schedule). Never rely on a fixed index — the matrix grows a column each month.
99
+ * @returns {Array<{name:string, index:number}>} in header order.
100
+ */
101
+ function findMonthColumns(header) {
102
+ const re = /^[A-Za-z]{3}\s+\d{4}$/;
103
+ return header
104
+ .map((h, i) => ({ name: (h || '').trim(), index: i }))
105
+ .filter(c => re.test(c.name));
106
+ }
107
+
108
+ /* ------------------------------------------------------------------ *
109
+ * ZIP (minimal, dependency-free) — enough to open an .xlsx
110
+ * ------------------------------------------------------------------ */
111
+
112
+ /** Unzip a Buffer via its central directory. → Map<name, Buffer>. Handles stored + deflate. */
113
+ function unzip(buf) {
114
+ const files = new Map();
115
+ // Locate End Of Central Directory (scan back from the end; max 64k comment).
116
+ let eocd = -1;
117
+ const minEnd = Math.max(0, buf.length - 22 - 0xffff);
118
+ for (let i = buf.length - 22; i >= minEnd; i--) {
119
+ if (buf.readUInt32LE(i) === 0x06054b50) { eocd = i; break; }
120
+ }
121
+ if (eocd < 0) throw new Error('filekit.unzip: not a zip (no EOCD signature)');
122
+ const count = buf.readUInt16LE(eocd + 10);
123
+ let p = buf.readUInt32LE(eocd + 16); // central directory offset
124
+ for (let n = 0; n < count; n++) {
125
+ if (buf.readUInt32LE(p) !== 0x02014b50) break; // central file header
126
+ const method = buf.readUInt16LE(p + 10);
127
+ const compSize = buf.readUInt32LE(p + 20);
128
+ const nameLen = buf.readUInt16LE(p + 28);
129
+ const extraLen = buf.readUInt16LE(p + 30);
130
+ const cmtLen = buf.readUInt16LE(p + 32);
131
+ const localOff = buf.readUInt32LE(p + 42);
132
+ const name = buf.toString('utf8', p + 46, p + 46 + nameLen);
133
+ // Jump to the local header to compute where the data actually starts.
134
+ const lNameLen = buf.readUInt16LE(localOff + 26);
135
+ const lExtraLen = buf.readUInt16LE(localOff + 28);
136
+ const dataStart = localOff + 30 + lNameLen + lExtraLen;
137
+ const comp = buf.subarray(dataStart, dataStart + compSize);
138
+ let content;
139
+ if (method === 0) content = Buffer.from(comp);
140
+ else if (method === 8) content = zlib.inflateRawSync(comp);
141
+ else throw new Error('filekit.unzip: unsupported compression method ' + method + ' for ' + name);
142
+ files.set(name, content);
143
+ p += 46 + nameLen + extraLen + cmtLen;
144
+ }
145
+ return files;
146
+ }
147
+
148
+ /* ------------------------------------------------------------------ *
149
+ * XLSX
150
+ * ------------------------------------------------------------------ */
151
+
152
+ function colToNum(letters) {
153
+ let n = 0;
154
+ for (const ch of letters) n = n * 26 + (ch.charCodeAt(0) - 64);
155
+ return n;
156
+ }
157
+
158
+ const XML_ENT = { '&amp;': '&', '&lt;': '<', '&gt;': '>', '&quot;': '"', '&apos;': "'", '&#10;': ' ', '&#39;': "'" };
159
+ function unescapeXml(s) {
160
+ return s.replace(/&amp;|&lt;|&gt;|&quot;|&apos;|&#10;|&#39;/g, m => XML_ENT[m]);
161
+ }
162
+
163
+ function parseSharedStrings(xml) {
164
+ const out = [];
165
+ let m; const re = /<si>([\s\S]*?)<\/si>/g;
166
+ while ((m = re.exec(xml))) {
167
+ const parts = [...m[1].matchAll(/<t[^>]*>([\s\S]*?)<\/t>/g)].map(x => x[1]);
168
+ out.push(unescapeXml(parts.join('')));
169
+ }
170
+ return out;
171
+ }
172
+
173
+ /** One worksheet: sparse cell grid with a 1-based Excel-style accessor. */
174
+ function makeSheet(xml, shared) {
175
+ const cells = Object.create(null); // cells[row][col] = value (1-based)
176
+ let maxRow = 0, maxCol = 0;
177
+ let rm; const rowRe = /<row[^>]*\br="(\d+)"[^>]*>([\s\S]*?)<\/row>/g;
178
+ while ((rm = rowRe.exec(xml))) {
179
+ const rn = +rm[1];
180
+ const line = (cells[rn] = cells[rn] || Object.create(null));
181
+ let cm; const cellRe = /<c r="([A-Z]+)(\d+)"([^>]*?)(?:\/>|>([\s\S]*?)<\/c>)/g;
182
+ while ((cm = cellRe.exec(rm[2]))) {
183
+ const col = colToNum(cm[1]);
184
+ const attrs = cm[3] || '';
185
+ const inner = cm[4] || '';
186
+ const t = (attrs.match(/t="([^"]*)"/) || [])[1];
187
+ let val = '';
188
+ const isMatch = inner.match(/<is>[\s\S]*?<t[^>]*>([\s\S]*?)<\/t>[\s\S]*?<\/is>/);
189
+ if (isMatch) val = unescapeXml(isMatch[1]); // inline string
190
+ else {
191
+ const v = (inner.match(/<v>([\s\S]*?)<\/v>/) || [])[1];
192
+ if (v !== undefined) val = (t === 's') ? (shared[+v] ?? '') : v; // shared str vs number/formula
193
+ }
194
+ line[col] = val;
195
+ if (col > maxCol) maxCol = col;
196
+ }
197
+ if (rn > maxRow) maxRow = rn;
198
+ }
199
+ return {
200
+ rowCount: maxRow,
201
+ colCount: maxCol,
202
+ /** 1-based, Excel-style: at(12, 3) === cell C12. */
203
+ at(row, col) { return (cells[row] && cells[row][col]) ?? ''; },
204
+ /** the row's cells as a 1-based-ish object (col → value). */
205
+ row(row) { return cells[row] || {}; },
206
+ };
207
+ }
208
+
209
+ /**
210
+ * Resolve the workbook's real sheet names, in workbook order, to their part paths.
211
+ * workbook.xml gives name + r:id; workbook.xml.rels maps r:id → worksheets/sheetN.xml.
212
+ * @returns {Array<{name:string, path:string}>}
213
+ */
214
+ function resolveSheets(files) {
215
+ const wbXml = (files.get('xl/workbook.xml') || Buffer.from('')).toString('utf8');
216
+ const relsXml = (files.get('xl/_rels/workbook.xml.rels') || Buffer.from('')).toString('utf8');
217
+
218
+ const rels = {}; // rId → target part path
219
+ for (const m of relsXml.matchAll(/<Relationship\b[^>]*>/g)) {
220
+ const id = (m[0].match(/Id="([^"]+)"/) || [])[1];
221
+ let target = (m[0].match(/Target="([^"]+)"/) || [])[1];
222
+ if (!id || !target) continue;
223
+ target = target.replace(/^\/?xl\//, '').replace(/^\.\//, '');
224
+ rels[id] = 'xl/' + target;
225
+ }
226
+
227
+ const out = [];
228
+ for (const m of wbXml.matchAll(/<sheet\b[^>]*\/?>/g)) {
229
+ const name = (m[0].match(/name="([^"]*)"/) || [])[1];
230
+ const rid = (m[0].match(/r:id="([^"]+)"/) || [])[1];
231
+ const path = rels[rid];
232
+ if (name && path && files.has(path)) out.push({ name: unescapeXml(name), path });
233
+ }
234
+ if (out.length) return out;
235
+
236
+ // Fallback: no workbook.xml — order sheetN.xml numerically, name them by index.
237
+ return [...files.keys()]
238
+ .filter(k => /^xl\/worksheets\/sheet\d+\.xml$/.test(k))
239
+ .sort((a, b) => (+a.match(/(\d+)/)[1]) - (+b.match(/(\d+)/)[1]))
240
+ .map((path, i) => ({ name: `Sheet${i + 1}`, path }));
241
+ }
242
+
243
+ /**
244
+ * Read an .xlsx directly (path or Buffer). Unzip is built in.
245
+ * @returns {{sheetNames:string[], sheet:(nameOrIndex?:string|number)=>Sheet, sharedStrings:string[]}}
246
+ * `sheet()` accepts a real sheet name ("GL 24500 Accrued Bonus") or a 0-based index.
247
+ */
248
+ function readXlsx(pathOrBuffer) {
249
+ const buf = Buffer.isBuffer(pathOrBuffer) ? pathOrBuffer : fs.readFileSync(pathOrBuffer);
250
+ const files = unzip(buf);
251
+ const ssXml = files.has('xl/sharedStrings.xml') ? files.get('xl/sharedStrings.xml').toString('utf8') : '';
252
+ const shared = parseSharedStrings(ssXml);
253
+ const sheets = resolveSheets(files);
254
+ const cache = {};
255
+ return {
256
+ sheetNames: sheets.map(s => s.name),
257
+ sharedStrings: shared,
258
+ sheet(nameOrIndex = 0) {
259
+ const i = (typeof nameOrIndex === 'number')
260
+ ? nameOrIndex
261
+ : sheets.findIndex(s => s.name === nameOrIndex);
262
+ if (i < 0 || !sheets[i]) throw new Error('filekit.readXlsx: no sheet ' + JSON.stringify(nameOrIndex));
263
+ if (!cache[i]) cache[i] = makeSheet(files.get(sheets[i].path).toString('utf8'), shared);
264
+ return cache[i];
265
+ },
266
+ };
267
+ }
268
+
269
+ /* ------------------------------------------------------------------ *
270
+ * XLSX writing — generate a reference workbook to hand a client.
271
+ * Stored (uncompressed) zip + the minimal OOXML parts. No dependencies.
272
+ * ------------------------------------------------------------------ */
273
+
274
+ const CRC_TABLE = (() => {
275
+ const t = new Int32Array(256);
276
+ for (let n = 0; n < 256; n++) {
277
+ let c = n;
278
+ for (let k = 0; k < 8; k++) c = (c & 1) ? (0xedb88320 ^ (c >>> 1)) : (c >>> 1);
279
+ t[n] = c;
280
+ }
281
+ return t;
282
+ })();
283
+
284
+ function crc32(buf) {
285
+ if (typeof zlib.crc32 === 'function') return zlib.crc32(buf) >>> 0;
286
+ let c = -1;
287
+ for (let i = 0; i < buf.length; i++) c = CRC_TABLE[(c ^ buf[i]) & 0xff] ^ (c >>> 8);
288
+ return (c ^ -1) >>> 0;
289
+ }
290
+
291
+ /** 1-based column number → Excel letters (1 → A, 27 → AA). */
292
+ function numToCol(n) {
293
+ let s = '';
294
+ while (n > 0) { const m = (n - 1) % 26; s = String.fromCharCode(65 + m) + s; n = Math.floor((n - 1) / 26); }
295
+ return s;
296
+ }
297
+
298
+ function escXml(s) {
299
+ return String(s)
300
+ .replace(/&/g, '&amp;').replace(/</g, '&lt;').replace(/>/g, '&gt;')
301
+ .replace(/"/g, '&quot;').replace(/\r/g, '')
302
+ // strip control chars XML 1.0 forbids
303
+ .replace(/[\x00-\x08\x0B\x0C\x0E-\x1F]/g, '');
304
+ }
305
+
306
+ /**
307
+ * Build a ZIP from Map<name, Buffer>. Deterministic: fixed timestamps, and each
308
+ * entry is deflated when that shrinks it, else stored — so output is byte-stable
309
+ * for a given input and stays comparable in size to the file it replaces.
310
+ */
311
+ function zipStore(files) {
312
+ const DOS_DATE = 33, DOS_TIME = 0; // 1980-01-01, so output is byte-stable
313
+ const locals = [], centrals = [];
314
+ let offset = 0;
315
+
316
+ for (const [name, data] of files) {
317
+ const nameBuf = Buffer.from(name, 'utf8');
318
+ const crc = crc32(data); // always over the UNCOMPRESSED bytes
319
+
320
+ const deflated = zlib.deflateRawSync(data, { level: 9 });
321
+ const useDeflate = deflated.length < data.length;
322
+ const method = useDeflate ? 8 : 0;
323
+ const payload = useDeflate ? deflated : data;
324
+
325
+ const local = Buffer.alloc(30);
326
+ local.writeUInt32LE(0x04034b50, 0);
327
+ local.writeUInt16LE(20, 4); // version needed
328
+ local.writeUInt16LE(0, 6); // flags
329
+ local.writeUInt16LE(method, 8);
330
+ local.writeUInt16LE(DOS_TIME, 10);
331
+ local.writeUInt16LE(DOS_DATE, 12);
332
+ local.writeUInt32LE(crc, 14);
333
+ local.writeUInt32LE(payload.length, 18); // compressed size
334
+ local.writeUInt32LE(data.length, 22); // uncompressed size
335
+ local.writeUInt16LE(nameBuf.length, 26);
336
+ local.writeUInt16LE(0, 28);
337
+ locals.push(local, nameBuf, payload);
338
+
339
+ const central = Buffer.alloc(46);
340
+ central.writeUInt32LE(0x02014b50, 0);
341
+ central.writeUInt16LE(20, 4); // version made by
342
+ central.writeUInt16LE(20, 6); // version needed
343
+ central.writeUInt16LE(0, 8);
344
+ central.writeUInt16LE(method, 10);
345
+ central.writeUInt16LE(DOS_TIME, 12);
346
+ central.writeUInt16LE(DOS_DATE, 14);
347
+ central.writeUInt32LE(crc, 16);
348
+ central.writeUInt32LE(payload.length, 20);
349
+ central.writeUInt32LE(data.length, 24);
350
+ central.writeUInt16LE(nameBuf.length, 28);
351
+ central.writeUInt16LE(0, 30); // extra
352
+ central.writeUInt16LE(0, 32); // comment
353
+ central.writeUInt16LE(0, 34); // disk
354
+ central.writeUInt16LE(0, 36); // internal attrs
355
+ central.writeUInt32LE(0, 38); // external attrs
356
+ central.writeUInt32LE(offset, 42);
357
+ centrals.push(central, nameBuf);
358
+
359
+ offset += local.length + nameBuf.length + payload.length;
360
+ }
361
+
362
+ const localPart = Buffer.concat(locals);
363
+ const centralPart = Buffer.concat(centrals);
364
+ const eocd = Buffer.alloc(22);
365
+ eocd.writeUInt32LE(0x06054b50, 0);
366
+ eocd.writeUInt16LE(0, 4);
367
+ eocd.writeUInt16LE(0, 6);
368
+ eocd.writeUInt16LE(files.size, 8);
369
+ eocd.writeUInt16LE(files.size, 10);
370
+ eocd.writeUInt32LE(centralPart.length, 12);
371
+ eocd.writeUInt32LE(localPart.length, 16);
372
+ eocd.writeUInt16LE(0, 20);
373
+
374
+ return Buffer.concat([localPart, centralPart, eocd]);
375
+ }
376
+
377
+ const SHEET_NAME_BAD = /[\[\]:*?/\\]/;
378
+
379
+ function sheetXml(rows) {
380
+ const out = [
381
+ '<?xml version="1.0" encoding="UTF-8" standalone="yes"?>',
382
+ '<worksheet xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main"><sheetData>',
383
+ ];
384
+ rows.forEach((row, r) => {
385
+ const cells = [];
386
+ row.forEach((v, c) => {
387
+ if (v === null || v === undefined || v === '') return; // empty cell = null on ingest
388
+ const ref = numToCol(c + 1) + (r + 1);
389
+ if (typeof v === 'number' && Number.isFinite(v)) {
390
+ cells.push(`<c r="${ref}"><v>${v}</v></c>`);
391
+ } else {
392
+ cells.push(`<c r="${ref}" t="inlineStr"><is><t xml:space="preserve">${escXml(v)}</t></is></c>`);
393
+ }
394
+ });
395
+ if (cells.length) out.push(`<row r="${r + 1}">${cells.join('')}</row>`);
396
+ });
397
+ out.push('</sheetData></worksheet>');
398
+ return Buffer.from(out.join(''), 'utf8');
399
+ }
400
+
401
+ /**
402
+ * Write a multi-tab .xlsx. One tab per dataset — the tab name becomes the bronze table suffix.
403
+ * @param {string|null} filePath write here, or null to only return the buffer
404
+ * @param {Array<{name:string, rows:Array<Array<string|number|null>>}>} sheets row 0 = header
405
+ * @returns {Buffer} the .xlsx bytes
406
+ */
407
+ function writeXlsx(filePath, sheets) {
408
+ if (!sheets || !sheets.length) throw new Error('filekit.writeXlsx: need at least one sheet');
409
+ for (const s of sheets) {
410
+ if (!s.name || s.name.length > 31 || SHEET_NAME_BAD.test(s.name)) {
411
+ throw new Error('filekit.writeXlsx: invalid sheet name ' + JSON.stringify(s.name));
412
+ }
413
+ }
414
+ const names = sheets.map(s => s.name);
415
+ if (new Set(names).size !== names.length) throw new Error('filekit.writeXlsx: duplicate sheet names');
416
+
417
+ const files = new Map();
418
+ const stylesRid = 'rId' + (sheets.length + 1);
419
+
420
+ files.set('[Content_Types].xml', Buffer.from(
421
+ '<?xml version="1.0" encoding="UTF-8" standalone="yes"?>' +
422
+ '<Types xmlns="http://schemas.openxmlformats.org/package/2006/content-types">' +
423
+ '<Default Extension="rels" ContentType="application/vnd.openxmlformats-package.relationships+xml"/>' +
424
+ '<Default Extension="xml" ContentType="application/xml"/>' +
425
+ '<Override PartName="/xl/workbook.xml" ContentType="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet.main+xml"/>' +
426
+ sheets.map((_, i) => `<Override PartName="/xl/worksheets/sheet${i + 1}.xml" ContentType="application/vnd.openxmlformats-officedocument.spreadsheetml.worksheet+xml"/>`).join('') +
427
+ '<Override PartName="/xl/styles.xml" ContentType="application/vnd.openxmlformats-officedocument.spreadsheetml.styles+xml"/>' +
428
+ '</Types>', 'utf8'));
429
+
430
+ files.set('_rels/.rels', Buffer.from(
431
+ '<?xml version="1.0" encoding="UTF-8" standalone="yes"?>' +
432
+ '<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships">' +
433
+ '<Relationship Id="rId1" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/officeDocument" Target="xl/workbook.xml"/>' +
434
+ '</Relationships>', 'utf8'));
435
+
436
+ files.set('xl/workbook.xml', Buffer.from(
437
+ '<?xml version="1.0" encoding="UTF-8" standalone="yes"?>' +
438
+ '<workbook xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main" ' +
439
+ 'xmlns:r="http://schemas.openxmlformats.org/officeDocument/2006/relationships"><sheets>' +
440
+ sheets.map((s, i) => `<sheet name="${escXml(s.name)}" sheetId="${i + 1}" r:id="rId${i + 1}"/>`).join('') +
441
+ '</sheets></workbook>', 'utf8'));
442
+
443
+ files.set('xl/_rels/workbook.xml.rels', Buffer.from(
444
+ '<?xml version="1.0" encoding="UTF-8" standalone="yes"?>' +
445
+ '<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships">' +
446
+ sheets.map((_, i) => `<Relationship Id="rId${i + 1}" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/worksheet" Target="worksheets/sheet${i + 1}.xml"/>`).join('') +
447
+ `<Relationship Id="${stylesRid}" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/styles" Target="styles.xml"/>` +
448
+ '</Relationships>', 'utf8'));
449
+
450
+ files.set('xl/styles.xml', Buffer.from(
451
+ '<?xml version="1.0" encoding="UTF-8" standalone="yes"?>' +
452
+ '<styleSheet xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main">' +
453
+ '<fonts count="1"><font><sz val="11"/><name val="Calibri"/></font></fonts>' +
454
+ '<fills count="2"><fill><patternFill patternType="none"/></fill><fill><patternFill patternType="gray125"/></fill></fills>' +
455
+ '<borders count="1"><border/></borders>' +
456
+ '<cellStyleXfs count="1"><xf numFmtId="0" fontId="0" fillId="0" borderId="0"/></cellStyleXfs>' +
457
+ '<cellXfs count="1"><xf numFmtId="0" fontId="0" fillId="0" borderId="0" xfId="0"/></cellXfs>' +
458
+ '</styleSheet>', 'utf8'));
459
+
460
+ sheets.forEach((s, i) => files.set(`xl/worksheets/sheet${i + 1}.xml`, sheetXml(s.rows)));
461
+
462
+ const buf = zipStore(files);
463
+ if (filePath) fs.writeFileSync(filePath, buf);
464
+ return buf;
465
+ }
466
+
467
+ /* ------------------------------------------------------------------ *
468
+ * Small grid helpers
469
+ * ------------------------------------------------------------------ */
470
+
471
+ /**
472
+ * Find the first Excel row (1-based) whose cells satisfy a predicate.
473
+ * @param {Sheet} sheet
474
+ * @param {(rowObj:object, rowNum:number)=>boolean} predicate
475
+ */
476
+ function findRow(sheet, predicate) {
477
+ for (let r = 1; r <= sheet.rowCount; r++) {
478
+ if (predicate(sheet.row(r), r)) return r;
479
+ }
480
+ return -1;
481
+ }
482
+
483
+ /** In a header row of Excel serial dates, find the column whose date is `ym` ("YYYY-MM"). */
484
+ function findColumnBySerialMonth(sheet, headerRowNum, ym) {
485
+ const line = sheet.row(headerRowNum);
486
+ for (const col of Object.keys(line)) {
487
+ const n = num(line[col]);
488
+ if (n > 40000 && n < 60000 && excelSerialToYM(n) === ym) return +col;
489
+ }
490
+ return -1;
491
+ }
492
+
493
+ module.exports = {
494
+ num, excelSerialToDate, excelSerialToYM,
495
+ parseCsv, tokenizeCsv, findMonthColumns,
496
+ unzip, readXlsx, resolveSheets, findRow, findColumnBySerialMonth,
497
+ zipStore, writeXlsx, numToCol, crc32,
498
+ };
package/package.json ADDED
@@ -0,0 +1,25 @@
1
+ {
2
+ "name": "@idea1/filekit",
3
+ "version": "1.0.0",
4
+ "description": "Messy-file reader/writer toolkit (CSV + XLSX) for the close kit and similar agent-driven data pipelines. Pure Node, no dependencies.",
5
+ "main": "filekit.js",
6
+ "files": [
7
+ "filekit.js",
8
+ "README.md"
9
+ ],
10
+ "engines": {
11
+ "node": ">=20"
12
+ },
13
+ "scripts": {
14
+ "test": "node filekit.test.js",
15
+ "build": "npm test",
16
+ "prepublishOnly": "npm run build",
17
+ "deploy": "npm publish --access public"
18
+ },
19
+ "repository": {
20
+ "type": "git",
21
+ "url": "git+https://github.com/datafabriq-co/datafabriq.git",
22
+ "directory": "packages/filekit"
23
+ },
24
+ "license": "MIT"
25
+ }