@vertesia/memory 0.78.0-dev-28b447d → 0.78.0-dev.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/cjs/Builder.js +186 -0
- package/lib/cjs/Builder.js.map +1 -0
- package/lib/cjs/ContentObject.js +114 -0
- package/lib/cjs/ContentObject.js.map +1 -0
- package/lib/cjs/ContentSource.js +82 -0
- package/lib/cjs/ContentSource.js.map +1 -0
- package/lib/cjs/MemoryPack.js +228 -0
- package/lib/cjs/MemoryPack.js.map +1 -0
- package/lib/cjs/MemoryPackBuilder.js +47 -0
- package/lib/cjs/MemoryPackBuilder.js.map +1 -0
- package/lib/cjs/commands/copy.js +53 -0
- package/lib/cjs/commands/copy.js.map +1 -0
- package/lib/cjs/commands/exec.js +82 -0
- package/lib/cjs/commands/exec.js.map +1 -0
- package/lib/cjs/index.js +28 -0
- package/lib/cjs/index.js.map +1 -0
- package/lib/cjs/package.json +3 -0
- package/lib/cjs/utils/cmdline.js +90 -0
- package/lib/cjs/utils/cmdline.js.map +1 -0
- package/lib/cjs/utils/rewrite.js +166 -0
- package/lib/cjs/utils/rewrite.js.map +1 -0
- package/lib/cjs/utils/stream.js +27 -0
- package/lib/cjs/utils/stream.js.map +1 -0
- package/lib/cjs/utils/tar.js +185 -0
- package/lib/cjs/utils/tar.js.map +1 -0
- package/lib/esm/Builder.js +178 -0
- package/lib/esm/Builder.js.map +1 -0
- package/lib/esm/ContentObject.js +103 -0
- package/lib/esm/ContentObject.js.map +1 -0
- package/lib/esm/ContentSource.js +75 -0
- package/lib/esm/ContentSource.js.map +1 -0
- package/lib/esm/MemoryPack.js +218 -0
- package/lib/esm/MemoryPack.js.map +1 -0
- package/lib/esm/MemoryPackBuilder.js +43 -0
- package/lib/esm/MemoryPackBuilder.js.map +1 -0
- package/lib/esm/commands/copy.js +50 -0
- package/lib/esm/commands/copy.js.map +1 -0
- package/lib/esm/commands/exec.js +75 -0
- package/lib/esm/commands/exec.js.map +1 -0
- package/lib/esm/index.js +7 -0
- package/lib/esm/index.js.map +1 -0
- package/lib/esm/utils/cmdline.js +86 -0
- package/lib/esm/utils/cmdline.js.map +1 -0
- package/lib/esm/utils/rewrite.js +161 -0
- package/lib/esm/utils/rewrite.js.map +1 -0
- package/lib/esm/utils/stream.js +23 -0
- package/lib/esm/utils/stream.js.map +1 -0
- package/lib/esm/utils/tar.js +175 -0
- package/lib/esm/utils/tar.js.map +1 -0
- package/lib/tsconfig.tsbuildinfo +1 -0
- package/lib/types/Builder.d.ts +73 -0
- package/lib/types/Builder.d.ts.map +1 -0
- package/lib/types/ContentObject.d.ts +44 -0
- package/lib/types/ContentObject.d.ts.map +1 -0
- package/lib/types/ContentSource.d.ts +33 -0
- package/lib/types/ContentSource.d.ts.map +1 -0
- package/lib/types/MemoryPack.d.ts +47 -0
- package/lib/types/MemoryPack.d.ts.map +1 -0
- package/lib/types/MemoryPackBuilder.d.ts +19 -0
- package/lib/types/MemoryPackBuilder.d.ts.map +1 -0
- package/lib/types/commands/copy.d.ts +9 -0
- package/lib/types/commands/copy.d.ts.map +1 -0
- package/lib/types/commands/exec.d.ts +8 -0
- package/lib/types/commands/exec.d.ts.map +1 -0
- package/lib/types/index.d.ts +15 -0
- package/lib/types/index.d.ts.map +1 -0
- package/lib/types/utils/cmdline.d.ts +11 -0
- package/lib/types/utils/cmdline.d.ts.map +1 -0
- package/lib/types/utils/rewrite.d.ts +39 -0
- package/lib/types/utils/rewrite.d.ts.map +1 -0
- package/lib/types/utils/stream.d.ts +10 -0
- package/lib/types/utils/stream.d.ts.map +1 -0
- package/lib/types/utils/tar.d.ts +41 -0
- package/lib/types/utils/tar.d.ts.map +1 -0
- package/package.json +43 -43
|
@@ -0,0 +1,185 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
+
};
|
|
5
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
exports.TarIndex = exports.TarBuilder = void 0;
|
|
7
|
+
exports.loadTarIndex = loadTarIndex;
|
|
8
|
+
exports.normalizePath = normalizePath;
|
|
9
|
+
const fs_1 = __importDefault(require("fs"));
|
|
10
|
+
const promises_1 = require("fs/promises");
|
|
11
|
+
const promises_2 = require("stream/promises");
|
|
12
|
+
const tar_stream_1 = __importDefault(require("tar-stream"));
|
|
13
|
+
const zlib_1 = __importDefault(require("zlib"));
|
|
14
|
+
class TarBuilder {
|
|
15
|
+
pack;
|
|
16
|
+
indexData = [];
|
|
17
|
+
currentOffset = 0;
|
|
18
|
+
tarPromise;
|
|
19
|
+
constructor(file) {
|
|
20
|
+
const pack = tar_stream_1.default.pack(); // Create a new tar stream
|
|
21
|
+
this.pack = pack;
|
|
22
|
+
// Open the output file as a write stream
|
|
23
|
+
const outputStream = fs_1.default.createWriteStream(file);
|
|
24
|
+
if (file.endsWith('.gz')) {
|
|
25
|
+
this.tarPromise = (0, promises_2.pipeline)(pack, zlib_1.default.createGzip(), outputStream);
|
|
26
|
+
}
|
|
27
|
+
else {
|
|
28
|
+
this.tarPromise = (0, promises_2.pipeline)(pack, outputStream);
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
async add(name, content) {
|
|
32
|
+
name = normalizePath(name);
|
|
33
|
+
// Calculate header size, 512 bytes for tar headers
|
|
34
|
+
const headerSize = 512;
|
|
35
|
+
const contentSize = content ? Buffer.byteLength(content) : 0;
|
|
36
|
+
const entryHeaderOffset = this.currentOffset;
|
|
37
|
+
// Store the index entry
|
|
38
|
+
// entry data offset is always at header offset + 512 bytes
|
|
39
|
+
if (contentSize > 0) { // do not index directories
|
|
40
|
+
this.indexData.push(`${name}:${entryHeaderOffset},${contentSize}`);
|
|
41
|
+
}
|
|
42
|
+
// Add the file entry to the tar stream
|
|
43
|
+
this.pack.entry({ name, size: contentSize }, content);
|
|
44
|
+
// Update the offset
|
|
45
|
+
this.currentOffset += headerSize + contentSize;
|
|
46
|
+
// Tar files are padded to 512-byte boundaries
|
|
47
|
+
if (contentSize % 512 !== 0) {
|
|
48
|
+
this.currentOffset += 512 - (contentSize % 512);
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
async build() {
|
|
52
|
+
const pack = this.pack;
|
|
53
|
+
// Convert index data to string and calculate its size
|
|
54
|
+
const indexContent = this.indexData.join('\n') + '\n';
|
|
55
|
+
const indexContentSize = Buffer.byteLength(indexContent);
|
|
56
|
+
// Add the .index entry to the tar
|
|
57
|
+
pack.entry({ name: '.index', size: indexContentSize }, indexContent);
|
|
58
|
+
pack.finalize(); // Finalize the tar stream
|
|
59
|
+
await this.tarPromise;
|
|
60
|
+
}
|
|
61
|
+
destroy() {
|
|
62
|
+
this.pack.destroy();
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
exports.TarBuilder = TarBuilder;
|
|
66
|
+
async function loadTarIndex(tarFile) {
|
|
67
|
+
const fd = await (0, promises_1.open)(tarFile, 'r');
|
|
68
|
+
try {
|
|
69
|
+
return await readTarIndex(fd);
|
|
70
|
+
}
|
|
71
|
+
catch (err) {
|
|
72
|
+
await fd.close();
|
|
73
|
+
throw err;
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
async function readTarIndex(fd) {
|
|
77
|
+
const stats = await fd.stat();
|
|
78
|
+
const size = stats.size;
|
|
79
|
+
// we want to find the index header.
|
|
80
|
+
// we read the last chunks of 512 until we find the file name followed by a 0 char.
|
|
81
|
+
// the tar file ends with a segment of 1024 bytes of 0 so we need to skip that.
|
|
82
|
+
// we pick a size for the buffer to also include the file size entry from the header. So the buffer should be
|
|
83
|
+
// of 100 + 8 + 8 + 8 + 12 bytes = 124 + 12 bytes = 136 bytes
|
|
84
|
+
// the file size will be located at offset 124 and is 12 bytes long
|
|
85
|
+
// skip 1024 0 bytes then skip another 1024 bytes to find the first possible location of the index header (512 bytes for content and 512 bytes for the header)
|
|
86
|
+
let offset = size - 1024 - 1024;
|
|
87
|
+
const buffer = Buffer.alloc(512);
|
|
88
|
+
while (offset >= 0) {
|
|
89
|
+
await fd.read(buffer, 0, 512, offset);
|
|
90
|
+
// remove the 0 byte padding
|
|
91
|
+
const fileName = buffer.toString('utf-8', 0, 100);
|
|
92
|
+
if (fileName.startsWith('.index\0')) {
|
|
93
|
+
// we found the index header
|
|
94
|
+
const indexSize = getHeaderFileSize(buffer);
|
|
95
|
+
const indexDataOffset = offset + 512;
|
|
96
|
+
const indexDataEnd = indexDataOffset + indexSize;
|
|
97
|
+
if (indexDataEnd > size - 1024) {
|
|
98
|
+
throw new Error('Invalid index data offsets: [' + indexDataOffset + ':' + indexDataEnd + ']');
|
|
99
|
+
}
|
|
100
|
+
const dataBuffer = Buffer.alloc(indexSize);
|
|
101
|
+
await fd.read(dataBuffer, 0, indexSize, indexDataOffset);
|
|
102
|
+
const indexContent = dataBuffer.toString('utf-8');
|
|
103
|
+
return new TarIndex(fd, indexContent);
|
|
104
|
+
}
|
|
105
|
+
offset -= 512;
|
|
106
|
+
}
|
|
107
|
+
return null;
|
|
108
|
+
}
|
|
109
|
+
class TarIndex {
|
|
110
|
+
fd;
|
|
111
|
+
entries = {};
|
|
112
|
+
headerBuffer = Buffer.alloc(512);
|
|
113
|
+
/**
|
|
114
|
+
* @param fd the tar file descriptor
|
|
115
|
+
* @param content the index content
|
|
116
|
+
*/
|
|
117
|
+
constructor(fd, content) {
|
|
118
|
+
this.fd = fd;
|
|
119
|
+
const lines = content.split('\n');
|
|
120
|
+
for (const line of lines) {
|
|
121
|
+
if (line) {
|
|
122
|
+
const [name, value] = line.split(':');
|
|
123
|
+
const [offsetStr, sizeStr] = value.split(',');
|
|
124
|
+
const offset = parseInt(offsetStr);
|
|
125
|
+
const size = parseInt(sizeStr);
|
|
126
|
+
this.entries[name] = { offset, size };
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
}
|
|
130
|
+
getPaths() {
|
|
131
|
+
return Object.keys(this.entries);
|
|
132
|
+
}
|
|
133
|
+
getSortedPaths() {
|
|
134
|
+
return Object.keys(this.entries).sort();
|
|
135
|
+
}
|
|
136
|
+
get(name) {
|
|
137
|
+
return this.entries[name];
|
|
138
|
+
}
|
|
139
|
+
async getContentAt(offset, size) {
|
|
140
|
+
const buffer = Buffer.alloc(size);
|
|
141
|
+
await this.fd.read(buffer, 0, size, offset + 512);
|
|
142
|
+
return buffer;
|
|
143
|
+
}
|
|
144
|
+
async getContent(name) {
|
|
145
|
+
const entry = this.entries[name];
|
|
146
|
+
if (entry) {
|
|
147
|
+
return this.getContentAt(entry.offset, entry.size);
|
|
148
|
+
}
|
|
149
|
+
else {
|
|
150
|
+
return null;
|
|
151
|
+
}
|
|
152
|
+
}
|
|
153
|
+
getReadStream(name, encoding) {
|
|
154
|
+
const entry = this.entries[name];
|
|
155
|
+
if (entry) {
|
|
156
|
+
const offset = entry.offset + 512;
|
|
157
|
+
return this.fd.createReadStream({
|
|
158
|
+
encoding,
|
|
159
|
+
start: entry.offset,
|
|
160
|
+
end: offset + entry.size
|
|
161
|
+
});
|
|
162
|
+
}
|
|
163
|
+
else {
|
|
164
|
+
return null;
|
|
165
|
+
}
|
|
166
|
+
}
|
|
167
|
+
async close() {
|
|
168
|
+
await this.fd.close();
|
|
169
|
+
}
|
|
170
|
+
}
|
|
171
|
+
exports.TarIndex = TarIndex;
|
|
172
|
+
function getHeaderFileSize(buffer) {
|
|
173
|
+
const octalSize = buffer.toString('ascii', 124, 136).trim();
|
|
174
|
+
return parseInt(octalSize, 8);
|
|
175
|
+
}
|
|
176
|
+
function normalizePath(path) {
|
|
177
|
+
if (path.startsWith('/')) {
|
|
178
|
+
path = path.slice(1);
|
|
179
|
+
}
|
|
180
|
+
if (path.endsWith('/')) {
|
|
181
|
+
path = path.slice(-1);
|
|
182
|
+
}
|
|
183
|
+
return path;
|
|
184
|
+
}
|
|
185
|
+
//# sourceMappingURL=tar.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"tar.js","sourceRoot":"","sources":["../../../src/utils/tar.ts"],"names":[],"mappings":";;;;;;AA0EA,oCAQC;AAgHD,sCAQC;AA1MD,4CAAoB;AACpB,0CAA+C;AAC/C,8CAA2C;AAC3C,4DAA6B;AAC7B,gDAAwB;AAOxB,MAAa,UAAU;IACnB,IAAI,CAAW;IACf,SAAS,GAAa,EAAE,CAAC;IACzB,aAAa,GAAG,CAAC,CAAC;IAClB,UAAU,CAAmB;IAE7B,YAAY,IAAY;QACpB,MAAM,IAAI,GAAG,oBAAG,CAAC,IAAI,EAAE,CAAC,CAAC,0BAA0B;QACnD,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;QACjB,yCAAyC;QACzC,MAAM,YAAY,GAAG,YAAE,CAAC,iBAAiB,CAAC,IAAI,CAAC,CAAC;QAChD,IAAI,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;YACvB,IAAI,CAAC,UAAU,GAAG,IAAA,mBAAQ,EAAC,IAAI,EAAE,cAAI,CAAC,UAAU,EAAE,EAAE,YAAY,CAAC,CAAC;QACtE,CAAC;aAAM,CAAC;YACJ,IAAI,CAAC,UAAU,GAAG,IAAA,mBAAQ,EAAC,IAAI,EAAE,YAAY,CAAC,CAAC;QACnD,CAAC;IACL,CAAC;IAGD,KAAK,CAAC,GAAG,CAAC,IAAY,EAAE,OAAgB;QACpC,IAAI,GAAG,aAAa,CAAC,IAAI,CAAC,CAAC;QAC3B,mDAAmD;QACnD,MAAM,UAAU,GAAG,GAAG,CAAC;QACvB,MAAM,WAAW,GAAG,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAC7D,MAAM,iBAAiB,GAAG,IAAI,CAAC,aAAa,CAAC;QAE7C,wBAAwB;QACxB,2DAA2D;QAC3D,IAAI,WAAW,GAAG,CAAC,EAAE,CAAC,CAAC,2BAA2B;YAC9C,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,IAAI,IAAI,iBAAiB,IAAI,WAAW,EAAE,CAAC,CAAC;QACvE,CAAC;QAED,uCAAuC;QACvC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,WAAW,EAAE,EAAE,OAAO,CAAC,CAAC;QAEtD,oBAAoB;QACpB,IAAI,CAAC,aAAa,IAAI,UAAU,GAAG,WAAW,CAAC;QAC/C,8CAA8C;QAC9C,IAAI,WAAW,GAAG,GAAG,KAAK,CAAC,EAAE,CAAC;YAC1B,IAAI,CAAC,aAAa,IAAI,GAAG,GAAG,CAAC,WAAW,GAAG,GAAG,CAAC,CAAC;QACpD,CAAC;IACL,CAAC;IAED,KAAK,CAAC,KAAK;QACP,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC;QACvB,sDAAsD;QACtD,MAAM,YAAY,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC;QACtD,MAAM,gBAAgB,GAAG,MAAM,CAAC,UAAU,CAAC,YAAY,CAAC,CAAC;QAEzD,kCAAkC;QAClC,IAAI,CAAC,KAAK,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,gBAAgB,EAAE,EAAE,YAAY,CAAC,CAAC;QAErE,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC,0BAA0B;QAE3C,MAAM,IAAI,CAAC,UAAU,CAAC;IAC1B,CAAC;IAED,OAAO;QACH,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC;IACxB,CAAC;CAEJ;AA7DD,gCA6DC;AAEM,KAAK,UAAU,YAAY,CAAC,OAAe;IAC9C,MAAM,EAAE,GAAG,MAAM,IAAA,eAAI,EAAC,OAAO,EAAE,GAAG,CAAC,CAAC;IACpC,IAAI,CAAC;QACD,OAAO,MAAM,YAAY,CAAC,EAAE,CAAC,CAAC;IAClC,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACX,MAAM,EAAE,CAAC,KAAK,EAAE,CAAC;QACjB,MAAM,GAAG,CAAC;IACd,CAAC;AACL,CAAC;AAED,KAAK,UAAU,YAAY,CAAC,EAAc;IACtC,MAAM,KAAK,GAAG,MAAM,EAAE,CAAC,IAAI,EAAE,CAAC;IAC9B,MAAM,IAAI,GAAG,KAAK,CAAC,IAAI,CAAC;IACxB,oCAAoC;IACpC,mFAAmF;IACnF,+EAA+E;IAC/E,6GAA6G;IAC7G,6DAA6D;IAC7D,mEAAmE;IACnE,8JAA8J;IAC9J,IAAI,MAAM,GAAG,IAAI,GAAG,IAAI,GAAG,IAAI,CAAC;IAChC,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IACjC,OAAO,MAAM,IAAI,CAAC,EAAE,CAAC;QACjB,MAAM,EAAE,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,GAAG,EAAE,MAAM,CAAC,CAAC;QACtC,4BAA4B;QAC5B,MAAM,QAAQ,GAAG,MAAM,CAAC,QAAQ,CAAC,OAAO,EAAE,CAAC,EAAE,GAAG,CAAC,CAAC;QAClD,IAAI,QAAQ,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;YAClC,4BAA4B;YAC5B,MAAM,SAAS,GAAG,iBAAiB,CAAC,MAAM,CAAC,CAAC;YAC5C,MAAM,eAAe,GAAG,MAAM,GAAG,GAAG,CAAC;YACrC,MAAM,YAAY,GAAG,eAAe,GAAG,SAAS,CAAC;YACjD,IAAI,YAAY,GAAG,IAAI,GAAG,IAAI,EAAE,CAAC;gBAC7B,MAAM,IAAI,KAAK,CAAC,+BAA+B,GAAG,eAAe,GAAG,GAAG,GAAG,YAAY,GAAG,GAAG,CAAC,CAAC;YAClG,CAAC;YACD,MAAM,UAAU,GAAG,MAAM,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;YAC3C,MAAM,EAAE,CAAC,IAAI,CAAC,UAAU,EAAE,CAAC,EAAE,SAAS,EAAE,eAAe,CAAC,CAAC;YACzD,MAAM,YAAY,GAAG,UAAU,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;YAClD,OAAO,IAAI,QAAQ,CAAC,EAAE,EAAE,YAAY,CAAC,CAAC;QAC1C,CAAC;QACD,MAAM,IAAI,GAAG,CAAC;IAClB,CAAC;IACD,OAAO,IAAI,CAAC;AAChB,CAAC;AAMD,MAAa,QAAQ;IAOE;IANnB,OAAO,GAAkC,EAAE,CAAC;IAC5C,YAAY,GAAG,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IACjC;;;OAGG;IACH,YAAmB,EAAc,EAAE,OAAe;QAA/B,OAAE,GAAF,EAAE,CAAY;QAC7B,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAClC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACvB,IAAI,IAAI,EAAE,CAAC;gBACP,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;gBACtC,MAAM,CAAC,SAAS,EAAE,OAAO,CAAC,GAAG,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;gBAC9C,MAAM,MAAM,GAAG,QAAQ,CAAC,SAAS,CAAC,CAAC;gBACnC,MAAM,IAAI,GAAG,QAAQ,CAAC,OAAO,CAAC,CAAC;gBAC/B,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;YAC1C,CAAC;QACL,CAAC;IACL,CAAC;IAED,QAAQ;QACJ,OAAO,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IACrC,CAAC;IAED,cAAc;QACV,OAAO,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC;IAC5C,CAAC;IAED,GAAG,CAAC,IAAY;QACZ,OAAO,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;IAC9B,CAAC;IAED,KAAK,CAAC,YAAY,CAAC,MAAc,EAAE,IAAY;QAC3C,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAClC,MAAM,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,GAAG,GAAG,CAAC,CAAC;QAClD,OAAO,MAAM,CAAC;IAClB,CAAC;IACD,KAAK,CAAC,UAAU,CAAC,IAAY;QACzB,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;QACjC,IAAI,KAAK,EAAE,CAAC;YACR,OAAO,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;QACvD,CAAC;aAAM,CAAC;YACJ,OAAO,IAAI,CAAC;QAChB,CAAC;IACL,CAAC;IAED,aAAa,CAAC,IAAY,EAAE,QAAyB;QACjD,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;QACjC,IAAI,KAAK,EAAE,CAAC;YACR,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM,GAAG,GAAG,CAAC;YAClC,OAAO,IAAI,CAAC,EAAE,CAAC,gBAAgB,CAAC;gBAC5B,QAAQ;gBACR,KAAK,EAAE,KAAK,CAAC,MAAM;gBACnB,GAAG,EAAE,MAAM,GAAG,KAAK,CAAC,IAAI;aAC3B,CAAC,CAAA;QACN,CAAC;aAAM,CAAC;YACJ,OAAO,IAAI,CAAC;QAChB,CAAC;IACL,CAAC;IAED,KAAK,CAAC,KAAK;QACP,MAAM,IAAI,CAAC,EAAE,CAAC,KAAK,EAAE,CAAC;IAC1B,CAAC;CAEJ;AAhED,4BAgEC;AAGD,SAAS,iBAAiB,CAAC,MAAc;IACrC,MAAM,SAAS,GAAG,MAAM,CAAC,QAAQ,CAAC,OAAO,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;IAC5D,OAAO,QAAQ,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC;AAClC,CAAC;AAED,SAAgB,aAAa,CAAC,IAAY;IACtC,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;QACvB,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IACzB,CAAC;IACD,IAAI,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;QACrB,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;IAC1B,CAAC;IACD,OAAO,IAAI,CAAC;AAChB,CAAC"}
|
|
@@ -0,0 +1,178 @@
|
|
|
1
|
+
import { AsyncObjectWalker } from "@vertesia/json";
|
|
2
|
+
import { mkdtempSync, rmSync } from "fs";
|
|
3
|
+
import os from "os";
|
|
4
|
+
import { join, resolve } from "path";
|
|
5
|
+
import { copy } from "./commands/copy.js";
|
|
6
|
+
import { exec } from "./commands/exec.js";
|
|
7
|
+
import { ContentObject, DocxObject, JsonObject, MediaObject, PdfObject } from "./ContentObject.js";
|
|
8
|
+
import { AbstractContentSource, TextSource } from "./ContentSource.js";
|
|
9
|
+
import { loadMemoryPack } from "./MemoryPack.js";
|
|
10
|
+
import { MemoryPackBuilder } from "./MemoryPackBuilder.js";
|
|
11
|
+
export class Builder {
|
|
12
|
+
options;
|
|
13
|
+
static instance;
|
|
14
|
+
static getInstance() {
|
|
15
|
+
if (!Builder.instance) {
|
|
16
|
+
throw new Error("No builder instance found");
|
|
17
|
+
}
|
|
18
|
+
return Builder.instance;
|
|
19
|
+
}
|
|
20
|
+
_vars;
|
|
21
|
+
_tmpdir;
|
|
22
|
+
memory;
|
|
23
|
+
constructor(options = {}) {
|
|
24
|
+
this.options = options;
|
|
25
|
+
this.memory = new MemoryPackBuilder(this);
|
|
26
|
+
this._vars = options.vars || {};
|
|
27
|
+
Builder.instance = this;
|
|
28
|
+
}
|
|
29
|
+
vars() {
|
|
30
|
+
return this._vars;
|
|
31
|
+
}
|
|
32
|
+
tmpdir() {
|
|
33
|
+
if (!this._tmpdir) {
|
|
34
|
+
this._tmpdir = mkdtempSync(join(os.tmpdir(), 'becomposable-memo-'));
|
|
35
|
+
}
|
|
36
|
+
return this._tmpdir;
|
|
37
|
+
}
|
|
38
|
+
from(location, options) {
|
|
39
|
+
return loadMemoryPack(location).then((memory) => {
|
|
40
|
+
return this.memory.load(memory, options);
|
|
41
|
+
});
|
|
42
|
+
}
|
|
43
|
+
addEntry(path, source) {
|
|
44
|
+
this.memory.add(path, source);
|
|
45
|
+
}
|
|
46
|
+
copy(location, path, options = {}) {
|
|
47
|
+
copy(this, location, path, options);
|
|
48
|
+
}
|
|
49
|
+
copyText(text, path) {
|
|
50
|
+
copy(this, new TextSource(text), path);
|
|
51
|
+
}
|
|
52
|
+
exec(cmd, options) {
|
|
53
|
+
return exec(cmd, options);
|
|
54
|
+
}
|
|
55
|
+
content(location, encoding) {
|
|
56
|
+
const source = AbstractContentSource.resolve(location);
|
|
57
|
+
if (Array.isArray(source)) {
|
|
58
|
+
return source.map(s => new ContentObject(this, s, encoding));
|
|
59
|
+
}
|
|
60
|
+
else {
|
|
61
|
+
return new ContentObject(this, source, encoding);
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
json(location) {
|
|
65
|
+
const source = AbstractContentSource.resolve(location);
|
|
66
|
+
if (Array.isArray(source)) {
|
|
67
|
+
return source.map(s => new JsonObject(this, s));
|
|
68
|
+
}
|
|
69
|
+
else {
|
|
70
|
+
return new JsonObject(this, source);
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
pdf(location) {
|
|
74
|
+
const source = AbstractContentSource.resolve(location);
|
|
75
|
+
if (Array.isArray(source)) {
|
|
76
|
+
return source.map(s => new PdfObject(this, s));
|
|
77
|
+
}
|
|
78
|
+
else {
|
|
79
|
+
return new PdfObject(this, source);
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
docx(location) {
|
|
83
|
+
const source = AbstractContentSource.resolve(location);
|
|
84
|
+
if (Array.isArray(source)) {
|
|
85
|
+
return source.map(s => new DocxObject(this, s));
|
|
86
|
+
}
|
|
87
|
+
else {
|
|
88
|
+
return new DocxObject(this, source);
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
media(location, options) {
|
|
92
|
+
const source = AbstractContentSource.resolve(location);
|
|
93
|
+
if (Array.isArray(source)) {
|
|
94
|
+
return source.map(s => new MediaObject(this, s, options));
|
|
95
|
+
}
|
|
96
|
+
else {
|
|
97
|
+
return new MediaObject(this, source, options);
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
async build(object) {
|
|
101
|
+
try {
|
|
102
|
+
let { fileName, publishName } = this._getOutputNames();
|
|
103
|
+
// resolve all content objects values from the context object
|
|
104
|
+
object = await resolveContextObject(object);
|
|
105
|
+
// write the memory to a file
|
|
106
|
+
fileName = await this.memory.build(fileName, object);
|
|
107
|
+
let target = fileName;
|
|
108
|
+
if (publishName) {
|
|
109
|
+
try {
|
|
110
|
+
target = await this.options.publish(fileName, publishName);
|
|
111
|
+
}
|
|
112
|
+
finally {
|
|
113
|
+
rmSync(fileName);
|
|
114
|
+
}
|
|
115
|
+
}
|
|
116
|
+
this.options.quiet || console.log(`Memory saved to ${target}`);
|
|
117
|
+
return target;
|
|
118
|
+
}
|
|
119
|
+
finally {
|
|
120
|
+
if (this._tmpdir) {
|
|
121
|
+
rmSync(this._tmpdir, { recursive: true });
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
_getOutputNames() {
|
|
126
|
+
const options = this.options;
|
|
127
|
+
if (!options.out) {
|
|
128
|
+
options.out = "memory.tar";
|
|
129
|
+
}
|
|
130
|
+
let fileName;
|
|
131
|
+
let publishName;
|
|
132
|
+
const out = options.out;
|
|
133
|
+
if (out.startsWith("memory:")) {
|
|
134
|
+
if (!options.publish) {
|
|
135
|
+
throw new Error(`The publish option is required for "${out}" output`);
|
|
136
|
+
}
|
|
137
|
+
// force gzip when publishing
|
|
138
|
+
options.gzip = true;
|
|
139
|
+
// create a temporary path for the output
|
|
140
|
+
fileName = createTmpBaseName(this.tmpdir());
|
|
141
|
+
publishName = out.substring("memory:".length);
|
|
142
|
+
}
|
|
143
|
+
else {
|
|
144
|
+
fileName = resolve(out);
|
|
145
|
+
}
|
|
146
|
+
return { fileName, publishName };
|
|
147
|
+
}
|
|
148
|
+
}
|
|
149
|
+
function resolveContextObject(object) {
|
|
150
|
+
return new AsyncObjectWalker().map(object, async (_key, value) => {
|
|
151
|
+
if (value instanceof ContentObject) {
|
|
152
|
+
return await value.getText();
|
|
153
|
+
}
|
|
154
|
+
else {
|
|
155
|
+
return value;
|
|
156
|
+
}
|
|
157
|
+
});
|
|
158
|
+
}
|
|
159
|
+
function createTmpBaseName(tmpdir) {
|
|
160
|
+
return join(tmpdir, `.composable-memory-${Date.now()}.tar`);
|
|
161
|
+
}
|
|
162
|
+
export function buildMemoryPack(recipeFn, options) {
|
|
163
|
+
const builder = new Builder(options);
|
|
164
|
+
return recipeFn({
|
|
165
|
+
vars: builder.vars.bind(builder),
|
|
166
|
+
tmpdir: builder.tmpdir.bind(builder),
|
|
167
|
+
exec: builder.exec.bind(builder),
|
|
168
|
+
from: builder.from.bind(builder),
|
|
169
|
+
content: builder.content.bind(builder),
|
|
170
|
+
json: builder.json.bind(builder),
|
|
171
|
+
pdf: builder.pdf.bind(builder),
|
|
172
|
+
docx: builder.docx.bind(builder),
|
|
173
|
+
media: builder.media.bind(builder),
|
|
174
|
+
copy: builder.copy.bind(builder),
|
|
175
|
+
copyText: builder.copyText.bind(builder),
|
|
176
|
+
}).then(metadata => builder.build(metadata));
|
|
177
|
+
}
|
|
178
|
+
//# sourceMappingURL=Builder.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"Builder.js","sourceRoot":"","sources":["../../src/Builder.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,MAAM,gBAAgB,CAAC;AACnD,OAAO,EAAE,WAAW,EAAE,MAAM,EAAE,MAAM,IAAI,CAAC;AACzC,OAAO,EAAE,MAAM,IAAI,CAAC;AACpB,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,MAAM,CAAC;AACrC,OAAO,EAAE,IAAI,EAAe,MAAM,oBAAoB,CAAC;AACvD,OAAO,EAAE,IAAI,EAAe,MAAM,oBAAoB,CAAC;AACvD,OAAO,EAAE,aAAa,EAAE,UAAU,EAAE,UAAU,EAAE,WAAW,EAAgB,SAAS,EAAE,MAAM,oBAAoB,CAAC;AACjH,OAAO,EAAE,qBAAqB,EAA6B,UAAU,EAAE,MAAM,oBAAoB,CAAC;AAClG,OAAO,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AACjD,OAAO,EAAe,iBAAiB,EAAE,MAAM,wBAAwB,CAAC;AAoDxE,MAAM,OAAO,OAAO;IAcG;IAbnB,MAAM,CAAC,QAAQ,CAAW;IAE1B,MAAM,CAAC,WAAW;QACd,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC;YACpB,MAAM,IAAI,KAAK,CAAC,2BAA2B,CAAC,CAAC;QACjD,CAAC;QACD,OAAO,OAAO,CAAC,QAAQ,CAAC;IAC5B,CAAC;IAED,KAAK,CAAsB;IAC3B,OAAO,CAAU;IACjB,MAAM,CAAoB;IAE1B,YAAmB,UAAwB,EAAE;QAA1B,YAAO,GAAP,OAAO,CAAmB;QACzC,IAAI,CAAC,MAAM,GAAG,IAAI,iBAAiB,CAAC,IAAI,CAAC,CAAC;QAC1C,IAAI,CAAC,KAAK,GAAG,OAAO,CAAC,IAAI,IAAI,EAAE,CAAC;QAChC,OAAO,CAAC,QAAQ,GAAG,IAAI,CAAC;IAC5B,CAAC;IAED,IAAI;QACA,OAAO,IAAI,CAAC,KAAK,CAAC;IACtB,CAAC;IAED,MAAM;QACF,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC;YAChB,IAAI,CAAC,OAAO,GAAG,WAAW,CAAC,IAAI,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,oBAAoB,CAAC,CAAC,CAAC;QACxE,CAAC;QACD,OAAO,IAAI,CAAC,OAAO,CAAC;IACxB,CAAC;IAED,IAAI,CAAC,QAAgB,EAAE,OAAqB;QACxC,OAAO,cAAc,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE;YAC5C,OAAO,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QAC7C,CAAC,CAAC,CAAC;IACP,CAAC;IAED,QAAQ,CAAC,IAAY,EAAE,MAAqB;QACxC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;IAClC,CAAC;IAED,IAAI,CAAC,QAAoB,EAAE,IAAY,EAAE,UAAuB,EAAE;QAC9D,IAAI,CAAC,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,OAAO,CAAC,CAAC;IACxC,CAAC;IAED,QAAQ,CAAC,IAAY,EAAE,IAAY;QAC/B,IAAI,CAAC,IAAI,EAAE,IAAI,UAAU,CAAC,IAAI,CAAC,EAAE,IAAI,CAAC,CAAC;IAC3C,CAAC;IAED,IAAI,CAAC,GAAW,EAAE,OAAqB;QACnC,OAAO,IAAI,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;IAC9B,CAAC;IAED,OAAO,CAAC,QAAgB,EAAE,QAAyB;QAC/C,MAAM,MAAM,GAAG,qBAAqB,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;QACvD,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;YACxB,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,IAAI,aAAa,CAAC,IAAI,EAAE,CAAC,EAAE,QAAQ,CAAC,CAAC,CAAC;QACjE,CAAC;aAAM,CAAC;YACJ,OAAO,IAAI,aAAa,CAAC,IAAI,EAAE,MAAM,EAAE,QAAQ,CAAC,CAAC;QACrD,CAAC;IACL,CAAC;IAED,IAAI,CAAC,QAAgB;QACjB,MAAM,MAAM,GAAG,qBAAqB,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;QACvD,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;YACxB,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,IAAI,UAAU,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;QACpD,CAAC;aAAM,CAAC;YACJ,OAAO,IAAI,UAAU,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;QACxC,CAAC;IACL,CAAC;IAED,GAAG,CAAC,QAAgB;QAChB,MAAM,MAAM,GAAG,qBAAqB,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;QACvD,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;YACxB,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,IAAI,SAAS,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;QACnD,CAAC;aAAM,CAAC;YACJ,OAAO,IAAI,SAAS,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;QACvC,CAAC;IACL,CAAC;IAED,IAAI,CAAC,QAAgB;QACjB,MAAM,MAAM,GAAG,qBAAqB,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;QACvD,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;YACxB,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,IAAI,UAAU,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;QACpD,CAAC;aAAM,CAAC;YACJ,OAAO,IAAI,UAAU,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;QACxC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,QAAgB,EAAE,OAAsB;QAC1C,MAAM,MAAM,GAAG,qBAAqB,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;QACvD,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;YACxB,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,IAAI,WAAW,CAAC,IAAI,EAAE,CAAC,EAAE,OAAO,CAAC,CAAC,CAAC;QAC9D,CAAC;aAAM,CAAC;YACJ,OAAO,IAAI,WAAW,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC;QAClD,CAAC;IACL,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,MAA2B;QACnC,IAAI,CAAC;YACD,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,GAAG,IAAI,CAAC,eAAe,EAAE,CAAC;YACvD,6DAA6D;YAC7D,MAAM,GAAG,MAAM,oBAAoB,CAAC,MAAM,CAAC,CAAC;YAC5C,6BAA6B;YAC7B,QAAQ,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;YACrD,IAAI,MAAM,GAAW,QAAQ,CAAC;YAC9B,IAAI,WAAW,EAAE,CAAC;gBACd,IAAI,CAAC;oBACD,MAAM,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,OAAQ,CAAC,QAAQ,EAAE,WAAW,CAAC,CAAC;gBAChE,CAAC;wBAAS,CAAC;oBACP,MAAM,CAAC,QAAQ,CAAC,CAAC;gBACrB,CAAC;YACL,CAAC;YACD,IAAI,CAAC,OAAO,CAAC,KAAK,IAAI,OAAO,CAAC,GAAG,CAAC,mBAAmB,MAAM,EAAE,CAAC,CAAC;YAC/D,OAAO,MAAM,CAAC;QAClB,CAAC;gBAAS,CAAC;YACP,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;gBACf,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;YAC9C,CAAC;QACL,CAAC;IACL,CAAC;IAEO,eAAe;QACnB,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC;QAC7B,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC;YACf,OAAO,CAAC,GAAG,GAAG,YAAY,CAAC;QAC/B,CAAC;QACD,IAAI,QAAgB,CAAC;QACrB,IAAI,WAA+B,CAAC;QACpC,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC;QACxB,IAAI,GAAG,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE,CAAC;YAC5B,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC;gBACnB,MAAM,IAAI,KAAK,CAAC,uCAAuC,GAAG,UAAU,CAAC,CAAC;YAC1E,CAAC;YACD,6BAA6B;YAC7B,OAAO,CAAC,IAAI,GAAG,IAAI,CAAC;YACpB,yCAAyC;YACzC,QAAQ,GAAG,iBAAiB,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;YAC5C,WAAW,GAAG,GAAG,CAAC,SAAS,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;QAClD,CAAC;aAAM,CAAC;YACJ,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC;QAC5B,CAAC;QACD,OAAO,EAAE,QAAQ,EAAE,WAAW,EAAE,CAAC;IACrC,CAAC;CAEJ;AAED,SAAS,oBAAoB,CAAC,MAA2B;IACrD,OAAO,IAAI,iBAAiB,EAAE,CAAC,GAAG,CAAC,MAAM,EAAE,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE;QAC7D,IAAI,KAAK,YAAY,aAAa,EAAE,CAAC;YACjC,OAAO,MAAM,KAAK,CAAC,OAAO,EAAE,CAAC;QACjC,CAAC;aAAM,CAAC;YACJ,OAAO,KAAK,CAAC;QACjB,CAAC;IACL,CAAC,CAAC,CAAC;AACP,CAAC;AAED,SAAS,iBAAiB,CAAC,MAAc;IACrC,OAAO,IAAI,CAAC,MAAM,EAAE,sBAAsB,IAAI,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;AAChE,CAAC;AAGD,MAAM,UAAU,eAAe,CAAC,QAA8D,EAAE,OAAqB;IACjH,MAAM,OAAO,GAAG,IAAI,OAAO,CAAC,OAAO,CAAC,CAAC;IACrC,OAAO,QAAQ,CAAC;QACZ,IAAI,EAAE,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC;QAChC,MAAM,EAAE,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC;QACpC,IAAI,EAAE,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC;QAChC,IAAI,EAAE,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC;QAChC,OAAO,EAAE,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC;QACtC,IAAI,EAAE,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC;QAChC,GAAG,EAAE,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC;QAC9B,IAAI,EAAE,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC;QAChC,KAAK,EAAE,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC;QAClC,IAAI,EAAE,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC;QAChC,QAAQ,EAAE,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC;KAC3C,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC;AACjD,CAAC"}
|
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
import { manyToMarkdown, pdfToText, pdfToTextBuffer, transformImage, transformImageToBuffer } from "@vertesia/converters";
|
|
2
|
+
import fs from "fs";
|
|
3
|
+
import { PassThrough, Readable } from "stream";
|
|
4
|
+
export class ContentObject {
|
|
5
|
+
builder;
|
|
6
|
+
source;
|
|
7
|
+
encoding;
|
|
8
|
+
constructor(builder, source, encoding = "utf-8") {
|
|
9
|
+
this.builder = builder;
|
|
10
|
+
this.source = source;
|
|
11
|
+
this.encoding = encoding;
|
|
12
|
+
}
|
|
13
|
+
getContent() {
|
|
14
|
+
return this.source.getContent();
|
|
15
|
+
}
|
|
16
|
+
getStream() {
|
|
17
|
+
return this.source.getStream();
|
|
18
|
+
}
|
|
19
|
+
copyTo(entry) {
|
|
20
|
+
this.builder.addEntry(entry, this);
|
|
21
|
+
}
|
|
22
|
+
async copyToFile(file) {
|
|
23
|
+
const input = await this.getStream();
|
|
24
|
+
const out = fs.createWriteStream(file);
|
|
25
|
+
const stream = input.pipe(out);
|
|
26
|
+
return new Promise((resolve, reject) => {
|
|
27
|
+
const handleError = (err) => {
|
|
28
|
+
reject(err);
|
|
29
|
+
out.close();
|
|
30
|
+
};
|
|
31
|
+
stream.on('finish', () => {
|
|
32
|
+
resolve(stream);
|
|
33
|
+
out.close();
|
|
34
|
+
});
|
|
35
|
+
input.on('error', handleError);
|
|
36
|
+
stream.on('error', handleError);
|
|
37
|
+
});
|
|
38
|
+
}
|
|
39
|
+
/**
|
|
40
|
+
* Get a text representation of the object
|
|
41
|
+
*/
|
|
42
|
+
async getText(encoding) {
|
|
43
|
+
const t = (await this.getContent()).toString(encoding || this.encoding);
|
|
44
|
+
return t;
|
|
45
|
+
}
|
|
46
|
+
getJsonValue() {
|
|
47
|
+
return this.getText();
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
export class JsonObject extends ContentObject {
|
|
51
|
+
async getJsonValue() {
|
|
52
|
+
return JSON.parse(await this.getText());
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
export class MediaObject extends ContentObject {
|
|
56
|
+
options;
|
|
57
|
+
constructor(builder, source, options = {}) {
|
|
58
|
+
super(builder, source);
|
|
59
|
+
this.options = options;
|
|
60
|
+
this.encoding = "base64";
|
|
61
|
+
}
|
|
62
|
+
async getStream() {
|
|
63
|
+
const stream = await super.getStream();
|
|
64
|
+
const out = new PassThrough();
|
|
65
|
+
await transformImage(stream, out, this.options);
|
|
66
|
+
return out;
|
|
67
|
+
}
|
|
68
|
+
async getContent() {
|
|
69
|
+
const stream = await super.getStream();
|
|
70
|
+
return await transformImageToBuffer(stream, this.options);
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
export class PdfObject extends ContentObject {
|
|
74
|
+
constructor(builder, source) {
|
|
75
|
+
super(builder, source);
|
|
76
|
+
}
|
|
77
|
+
async getStream() {
|
|
78
|
+
return Readable.from(await this.getContent());
|
|
79
|
+
}
|
|
80
|
+
async getContent() {
|
|
81
|
+
return pdfToTextBuffer(await super.getContent());
|
|
82
|
+
}
|
|
83
|
+
async getText() {
|
|
84
|
+
return await pdfToText(await super.getContent());
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
export class DocxObject extends ContentObject {
|
|
88
|
+
constructor(builder, source) {
|
|
89
|
+
super(builder, source);
|
|
90
|
+
}
|
|
91
|
+
async getBuffer() {
|
|
92
|
+
return Buffer.from(await manyToMarkdown(await this.getStream(), "docx"), "utf-8");
|
|
93
|
+
}
|
|
94
|
+
async getStream() {
|
|
95
|
+
const stream = new PassThrough();
|
|
96
|
+
stream.end(await this.getBuffer());
|
|
97
|
+
return stream;
|
|
98
|
+
}
|
|
99
|
+
async getText() {
|
|
100
|
+
return await manyToMarkdown(await this.getStream(), "docx");
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
//# sourceMappingURL=ContentObject.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"ContentObject.js","sourceRoot":"","sources":["../../src/ContentObject.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAE,SAAS,EAAE,eAAe,EAAE,cAAc,EAAE,sBAAsB,EAAE,MAAM,sBAAsB,CAAC;AAC1H,OAAO,EAAE,MAAM,IAAI,CAAC;AACpB,OAAO,EAAE,WAAW,EAAE,QAAQ,EAAE,MAAM,QAAQ,CAAC;AAI/C,MAAM,OAAO,aAAa;IACH;IAAyB;IAA8B;IAA1E,YAAmB,OAAgB,EAAS,MAAqB,EAAS,WAA2B,OAAO;QAAzF,YAAO,GAAP,OAAO,CAAS;QAAS,WAAM,GAAN,MAAM,CAAe;QAAS,aAAQ,GAAR,QAAQ,CAA0B;IAAI,CAAC;IAEjH,UAAU;QACN,OAAO,IAAI,CAAC,MAAM,CAAC,UAAU,EAAE,CAAC;IACpC,CAAC;IAED,SAAS;QACL,OAAO,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE,CAAC;IACnC,CAAC;IAED,MAAM,CAAC,KAAa;QAChB,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;IACvC,CAAC;IAED,KAAK,CAAC,UAAU,CAAC,IAAY;QAEzB,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,SAAS,EAAE,CAAC;QACrC,MAAM,GAAG,GAAG,EAAE,CAAC,iBAAiB,CAAC,IAAI,CAAC,CAAC;QACvC,MAAM,MAAM,GAAG,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAC/B,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACnC,MAAM,WAAW,GAAG,CAAC,GAAQ,EAAE,EAAE;gBAC7B,MAAM,CAAC,GAAG,CAAC,CAAC;gBACZ,GAAG,CAAC,KAAK,EAAE,CAAC;YAChB,CAAC,CAAA;YACD,MAAM,CAAC,EAAE,CAAC,QAAQ,EAAE,GAAG,EAAE;gBACrB,OAAO,CAAC,MAAM,CAAC,CAAC;gBAChB,GAAG,CAAC,KAAK,EAAE,CAAC;YAChB,CAAC,CAAC,CAAC;YACH,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,WAAW,CAAC,CAAC;YAC/B,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,WAAW,CAAC,CAAC;QACpC,CAAC,CAAC,CAAA;IACN,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,OAAO,CAAC,QAAyB;QACnC,MAAM,CAAC,GAAG,CAAC,MAAM,IAAI,CAAC,UAAU,EAAE,CAAC,CAAC,QAAQ,CAAC,QAAQ,IAAI,IAAI,CAAC,QAAQ,CAAC,CAAC;QACxE,OAAO,CAAC,CAAC;IACb,CAAC;IAED,YAAY;QACR,OAAO,IAAI,CAAC,OAAO,EAAE,CAAC;IAC1B,CAAC;CACJ;AAED,MAAM,OAAO,UAAW,SAAQ,aAAa;IACzC,KAAK,CAAC,YAAY;QACd,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC;IAC5C,CAAC;CACJ;AAMD,MAAM,OAAO,WAAY,SAAQ,aAAa;IACkB;IAA5D,YAAY,OAAgB,EAAE,MAAqB,EAAS,UAAwB,EAAE;QAClF,KAAK,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;QADiC,YAAO,GAAP,OAAO,CAAmB;QAElF,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;IAC7B,CAAC;IACD,KAAK,CAAC,SAAS;QACX,MAAM,MAAM,GAAG,MAAM,KAAK,CAAC,SAAS,EAAE,CAAC;QACvC,MAAM,GAAG,GAAG,IAAI,WAAW,EAAE,CAAC;QAC9B,MAAM,cAAc,CAAC,MAAM,EAAE,GAAG,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;QAChD,OAAO,GAAG,CAAC;IACf,CAAC;IACD,KAAK,CAAC,UAAU;QACZ,MAAM,MAAM,GAAG,MAAM,KAAK,CAAC,SAAS,EAAE,CAAC;QACvC,OAAO,MAAM,sBAAsB,CAAC,MAAM,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;IAC9D,CAAC;CACJ;AAGD,MAAM,OAAO,SAAU,SAAQ,aAAa;IACxC,YAAY,OAAgB,EAAE,MAAqB;QAC/C,KAAK,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;IAC3B,CAAC;IACD,KAAK,CAAC,SAAS;QACX,OAAO,QAAQ,CAAC,IAAI,CAAC,MAAM,IAAI,CAAC,UAAU,EAAE,CAAC,CAAC;IAClD,CAAC;IACD,KAAK,CAAC,UAAU;QACZ,OAAO,eAAe,CAAC,MAAM,KAAK,CAAC,UAAU,EAAE,CAAC,CAAC;IACrD,CAAC;IACD,KAAK,CAAC,OAAO;QACT,OAAO,MAAM,SAAS,CAAC,MAAM,KAAK,CAAC,UAAU,EAAE,CAAC,CAAC;IACrD,CAAC;CACJ;AAGD,MAAM,OAAO,UAAW,SAAQ,aAAa;IACzC,YAAY,OAAgB,EAAE,MAAqB;QAC/C,KAAK,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;IAC3B,CAAC;IACD,KAAK,CAAC,SAAS;QACX,OAAO,MAAM,CAAC,IAAI,CAAC,MAAM,cAAc,CAAC,MAAM,IAAI,CAAC,SAAS,EAAE,EAAE,MAAM,CAAC,EAAE,OAAO,CAAC,CAAC;IACtF,CAAC;IACD,KAAK,CAAC,SAAS;QACX,MAAM,MAAM,GAAG,IAAI,WAAW,EAAE,CAAC;QACjC,MAAM,CAAC,GAAG,CAAC,MAAM,IAAI,CAAC,SAAS,EAAE,CAAC,CAAC;QACnC,OAAO,MAAM,CAAC;IAClB,CAAC;IACD,KAAK,CAAC,OAAO;QACT,OAAO,MAAM,cAAc,CAAC,MAAM,IAAI,CAAC,SAAS,EAAE,EAAE,MAAM,CAAC,CAAC;IAChE,CAAC;CAEJ"}
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
import { createReadStream } from "fs";
|
|
2
|
+
import { readFile } from "fs/promises";
|
|
3
|
+
import { globSync } from 'glob';
|
|
4
|
+
import { basename, extname, resolve } from "path";
|
|
5
|
+
import { Readable } from "stream";
|
|
6
|
+
export class AbstractContentSource {
|
|
7
|
+
async getStream() {
|
|
8
|
+
return Readable.from(await this.getContent());
|
|
9
|
+
}
|
|
10
|
+
static resolve(source) {
|
|
11
|
+
if (typeof source === 'string') {
|
|
12
|
+
return FileSource.resolve(source);
|
|
13
|
+
}
|
|
14
|
+
else if (source instanceof AbstractContentSource) {
|
|
15
|
+
return source;
|
|
16
|
+
}
|
|
17
|
+
throw new Error("Unsupported content source: " + source);
|
|
18
|
+
}
|
|
19
|
+
}
|
|
20
|
+
export class BufferSource extends AbstractContentSource {
|
|
21
|
+
buffer;
|
|
22
|
+
constructor(buffer) {
|
|
23
|
+
super();
|
|
24
|
+
this.buffer = buffer;
|
|
25
|
+
}
|
|
26
|
+
getContent() {
|
|
27
|
+
return Promise.resolve(this.buffer);
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
export class TextSource extends AbstractContentSource {
|
|
31
|
+
value;
|
|
32
|
+
encoding;
|
|
33
|
+
constructor(value, encoding = "utf-8") {
|
|
34
|
+
super();
|
|
35
|
+
this.value = value;
|
|
36
|
+
this.encoding = encoding;
|
|
37
|
+
}
|
|
38
|
+
getContent() {
|
|
39
|
+
return Promise.resolve(Buffer.from(this.value, this.encoding));
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
export class FileSource extends AbstractContentSource {
|
|
43
|
+
file;
|
|
44
|
+
constructor(file, resolvePath = true) {
|
|
45
|
+
super();
|
|
46
|
+
this.file = resolvePath ? resolve(file) : file;
|
|
47
|
+
}
|
|
48
|
+
get path() {
|
|
49
|
+
return this.file;
|
|
50
|
+
}
|
|
51
|
+
get name() {
|
|
52
|
+
return basename(this.file);
|
|
53
|
+
}
|
|
54
|
+
get extname() {
|
|
55
|
+
return extname(this.file);
|
|
56
|
+
}
|
|
57
|
+
get nameWithoutExt() {
|
|
58
|
+
return this.name.slice(0, -this.extname.length);
|
|
59
|
+
}
|
|
60
|
+
getContent() {
|
|
61
|
+
return readFile(this.file);
|
|
62
|
+
}
|
|
63
|
+
async getStream() {
|
|
64
|
+
return createReadStream(this.file);
|
|
65
|
+
}
|
|
66
|
+
static resolve(location) {
|
|
67
|
+
if (location.includes('*')) {
|
|
68
|
+
return globSync(location, { absolute: true, withFileTypes: false }).map(f => new FileSource(f));
|
|
69
|
+
}
|
|
70
|
+
else {
|
|
71
|
+
return new FileSource(location);
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
//# sourceMappingURL=ContentSource.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"ContentSource.js","sourceRoot":"","sources":["../../src/ContentSource.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,gBAAgB,EAAE,MAAM,IAAI,CAAC;AACtC,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AACvC,OAAO,EAAE,QAAQ,EAAE,MAAM,MAAM,CAAC;AAChC,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,MAAM,CAAC;AAClD,OAAO,EAAE,QAAQ,EAAE,MAAM,QAAQ,CAAC;AAQlC,MAAM,OAAgB,qBAAqB;IAGvC,KAAK,CAAC,SAAS;QACX,OAAO,QAAQ,CAAC,IAAI,CAAC,MAAM,IAAI,CAAC,UAAU,EAAE,CAAC,CAAC;IAClD,CAAC;IAED,MAAM,CAAC,OAAO,CAAC,MAAkB;QAC7B,IAAI,OAAO,MAAM,KAAK,QAAQ,EAAE,CAAC;YAC7B,OAAO,UAAU,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;QACtC,CAAC;aAAM,IAAI,MAAM,YAAY,qBAAqB,EAAE,CAAC;YACjD,OAAO,MAAM,CAAC;QAClB,CAAC;QACD,MAAM,IAAI,KAAK,CAAC,8BAA8B,GAAG,MAAM,CAAC,CAAC;IAC7D,CAAC;CACJ;AAED,MAAM,OAAO,YAAa,SAAQ,qBAAqB;IAChC;IAAnB,YAAmB,MAAc;QAC7B,KAAK,EAAE,CAAA;QADQ,WAAM,GAAN,MAAM,CAAQ;IAEjC,CAAC;IACD,UAAU;QACN,OAAO,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IACxC,CAAC;CACJ;AAED,MAAM,OAAO,UAAW,SAAQ,qBAAqB;IAC9B;IAAsB;IAAzC,YAAmB,KAAa,EAAS,WAA2B,OAAO;QACvE,KAAK,EAAE,CAAC;QADO,UAAK,GAAL,KAAK,CAAQ;QAAS,aAAQ,GAAR,QAAQ,CAA0B;IAE3E,CAAC;IAED,UAAU;QACN,OAAO,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC;IACnE,CAAC;CACJ;AAED,MAAM,OAAO,UAAW,SAAQ,qBAAqB;IACjD,IAAI,CAAS;IACb,YAAY,IAAY,EAAE,WAAW,GAAG,IAAI;QACxC,KAAK,EAAE,CAAC;QACR,IAAI,CAAC,IAAI,GAAG,WAAW,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IACnD,CAAC;IAED,IAAI,IAAI;QACJ,OAAO,IAAI,CAAC,IAAI,CAAC;IACrB,CAAC;IAED,IAAI,IAAI;QACJ,OAAO,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC/B,CAAC;IAED,IAAI,OAAO;QACP,OAAO,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC9B,CAAC;IAED,IAAI,cAAc;QACd,OAAO,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;IACpD,CAAC;IAED,UAAU;QACN,OAAO,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC/B,CAAC;IAED,KAAK,CAAC,SAAS;QACX,OAAO,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACvC,CAAC;IAED,MAAM,CAAC,OAAO,CAAC,QAAgB;QAC3B,IAAI,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;YACzB,OAAO,QAAQ,CAAC,QAAQ,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,aAAa,EAAE,KAAK,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,IAAI,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC;QACpG,CAAC;aAAM,CAAC;YACJ,OAAO,IAAI,UAAU,CAAC,QAAQ,CAAC,CAAC;QACpC,CAAC;IACL,CAAC;CAEJ"}
|