@goatlab/typesense 0.1.4 → 0.1.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/TypesenseApi.d.ts +1 -1
- package/dist/actions/documents/exportDocuments.d.ts +1 -2
- package/dist/actions/documents/exportDocuments.js +1 -6
- package/dist/actions/documents/exportDocuments.js.map +1 -1
- package/dist/actions/documents/importDocuments.d.ts +1 -2
- package/dist/actions/documents/importDocuments.js +8 -9
- package/dist/actions/documents/importDocuments.js.map +1 -1
- package/dist/components/export-formatter.d.ts +7 -8
- package/dist/components/export-formatter.js +54 -72
- package/dist/components/export-formatter.js.map +1 -1
- package/package.json +1 -1
package/dist/TypesenseApi.d.ts
CHANGED
|
@@ -153,7 +153,7 @@ export declare class TypesenseApi<TDoc extends Record<string, any> = Record<stri
|
|
|
153
153
|
}, options?: TypesenseCollectionOptions) => Promise<TypesenseDocument<TDoc>>;
|
|
154
154
|
delete: (id: string | number, options?: TypesenseCollectionOptions) => any;
|
|
155
155
|
getById: (id: string | number, options?: TypesenseCollectionOptions) => any;
|
|
156
|
-
import: (documents: string |
|
|
156
|
+
import: (documents: string | ReadableStream<any> | TypesenseDocument<Record<string, any>>[], format?: import("./typesense.model").TypesenseImportFormat, importOptions?: import("./typesense.model").TypesenseImportOptions, collectionOptions?: TypesenseCollectionOptions) => any;
|
|
157
157
|
export: (format?: import("./typesense.model").TypesenseExportFormat, options?: import("./typesense.model").TypesenseExportOptions & TypesenseCollectionOptions) => any;
|
|
158
158
|
exportStream: (options?: import("./typesense.model").TypesenseExportOptions & TypesenseCollectionOptions) => any;
|
|
159
159
|
deleteByFilter: (filter: string, options?: import("./typesense.model").TypesenseDeleteByFilterOptions & TypesenseCollectionOptions) => any;
|
|
@@ -1,5 +1,4 @@
|
|
|
1
|
-
import { Readable } from 'node:stream';
|
|
2
1
|
import type { TypesenseContext } from '../../types';
|
|
3
2
|
import type { TypesenseCollectionOptions, TypesenseDocument, TypesenseExportFormat, TypesenseExportOptions } from '../../typesense.model';
|
|
4
3
|
export declare function exportDocuments<T extends Record<string, any>>(ctx: TypesenseContext, format?: TypesenseExportFormat, options?: TypesenseExportOptions & TypesenseCollectionOptions): Promise<string | TypesenseDocument<T>[]>;
|
|
5
|
-
export declare function exportDocumentsStream<_T extends Record<string, any>>(ctx: TypesenseContext, options?: TypesenseExportOptions & TypesenseCollectionOptions): Promise<
|
|
4
|
+
export declare function exportDocumentsStream<_T extends Record<string, any>>(ctx: TypesenseContext, options?: TypesenseExportOptions & TypesenseCollectionOptions): Promise<ReadableStream>;
|
|
@@ -2,7 +2,6 @@
|
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.exportDocuments = exportDocuments;
|
|
4
4
|
exports.exportDocumentsStream = exportDocumentsStream;
|
|
5
|
-
const node_stream_1 = require("node:stream");
|
|
6
5
|
const export_formatter_1 = require("../../components/export-formatter");
|
|
7
6
|
const typesense_model_1 = require("../../typesense.model");
|
|
8
7
|
async function exportDocuments(ctx, format = 'jsonl', options) {
|
|
@@ -42,10 +41,6 @@ async function exportDocumentsStream(ctx, options) {
|
|
|
42
41
|
const searchParams = {
|
|
43
42
|
...exportOptions,
|
|
44
43
|
};
|
|
45
|
-
return ctx.httpClient
|
|
46
|
-
.stream(`/collections/${collectionName}/documents/export`, {
|
|
47
|
-
searchParams,
|
|
48
|
-
})
|
|
49
|
-
.then(stream => node_stream_1.Readable.fromWeb(stream));
|
|
44
|
+
return ctx.httpClient.stream(`/collections/${collectionName}/documents/export`, { searchParams });
|
|
50
45
|
}
|
|
51
46
|
//# sourceMappingURL=exportDocuments.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"exportDocuments.js","sourceRoot":"","sources":["../../../src/actions/documents/exportDocuments.ts"],"names":[],"mappings":";;
|
|
1
|
+
{"version":3,"file":"exportDocuments.js","sourceRoot":"","sources":["../../../src/actions/documents/exportDocuments.ts"],"names":[],"mappings":";;AAUA,0CA0CC;AAED,sDAcC;AApED,wEAAmE;AAQnE,2DAAsD;AAE/C,KAAK,UAAU,eAAe,CACnC,GAAqB,EACrB,SAAgC,OAAO,EACvC,OAA6D;IAE7D,kBAAkB;IAClB,MAAM,gBAAgB,GAA4B,CAAC,OAAO,EAAE,MAAM,EAAE,KAAK,CAAC,CAAA;IAC1E,IAAI,CAAC,gBAAgB,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;QACvC,MAAM,IAAI,gCAAc,CAAC,8BAA8B,MAAM,EAAE,EAAE,GAAG,CAAC,CAAA;IACvE,CAAC;IAED,MAAM,cAAc,GAAG,OAAO,EAAE,UAAU,IAAI,GAAG,CAAC,IAAI,EAAE,CAAA;IACxD,MAAM,EAAE,UAAU,EAAE,CAAC,EAAE,GAAG,aAAa,EAAE,GAAG,OAAO,IAAI,EAAE,CAAA;IACzD,MAAM,YAAY,GAAQ;QACxB,GAAG,aAAa;QAChB,yEAAyE;KAC1E,CAAA;IAED,sCAAsC;IACtC,MAAM,QAAQ,GAAG,MAAM,GAAG,CAAC,UAAU,CAAC,WAAW,CAC/C,gBAAgB,cAAc,mBAAmB,EACjD,EAAE,YAAY,EAAE,CACjB,CAAA;IAED,IAAI,MAAM,KAAK,MAAM,EAAE,CAAC;QACtB,4BAA4B;QAC5B,OAAO,QAAQ;aACZ,KAAK,CAAC,IAAI,CAAC;aACX,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;aAC3B,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAA;IAClC,CAAC;IACD,IAAI,MAAM,KAAK,KAAK,EAAE,CAAC;QACrB,2CAA2C;QAC3C,MAAM,SAAS,GAAG,QAAQ;aACvB,KAAK,CAAC,IAAI,CAAC;aACX,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;aAC3B,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAA;QAEhC,OAAO,kCAAe,CAAC,SAAS,CAAC,SAAS,CAAC,CAAA;IAC7C,CAAC;IAED,OAAO,QAAQ,CAAA;AACjB,CAAC;AAEM,KAAK,UAAU,qBAAqB,CACzC,GAAqB,EACrB,OAA6D;IAE7D,MAAM,cAAc,GAAG,OAAO,EAAE,UAAU,IAAI,GAAG,CAAC,IAAI,EAAE,CAAA;IACxD,MAAM,EAAE,UAAU,EAAE,CAAC,EAAE,GAAG,aAAa,EAAE,GAAG,OAAO,IAAI,EAAE,CAAA;IACzD,MAAM,YAAY,GAAQ;QACxB,GAAG,aAAa;KACjB,CAAA;IAED,OAAO,GAAG,CAAC,UAAU,CAAC,MAAM,CAC1B,gBAAgB,cAAc,mBAAmB,EACjD,EAAE,YAAY,EAAE,CACjB,CAAA;AACH,CAAC"}
|
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
import { Readable } from 'node:stream';
|
|
2
1
|
import type { TypesenseContext } from '../../types';
|
|
3
2
|
import type { TypesenseCollectionOptions, TypesenseDocument, TypesenseImportFormat, TypesenseImportOptions, TypesenseImportResult } from '../../typesense.model';
|
|
4
|
-
export declare function importDocuments<T extends Record<string, any>>(ctx: TypesenseContext, documents: TypesenseDocument<T>[] | string |
|
|
3
|
+
export declare function importDocuments<T extends Record<string, any>>(ctx: TypesenseContext, documents: TypesenseDocument<T>[] | string | ReadableStream, format?: TypesenseImportFormat, importOptions?: TypesenseImportOptions, collectionOptions?: TypesenseCollectionOptions): Promise<TypesenseImportResult[]>;
|
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.importDocuments = importDocuments;
|
|
4
|
-
const node_stream_1 = require("node:stream");
|
|
5
4
|
const export_formatter_1 = require("../../components/export-formatter");
|
|
6
5
|
const typesense_model_1 = require("../../typesense.model");
|
|
7
6
|
async function importDocuments(ctx, documents, format = 'jsonl', importOptions, collectionOptions) {
|
|
@@ -11,9 +10,9 @@ async function importDocuments(ctx, documents, format = 'jsonl', importOptions,
|
|
|
11
10
|
throw new typesense_model_1.TypesenseError('Unsupported format', 400);
|
|
12
11
|
}
|
|
13
12
|
const collectionName = collectionOptions?.collection || ctx.fqcn();
|
|
14
|
-
let
|
|
15
|
-
if (documents instanceof
|
|
16
|
-
|
|
13
|
+
let body;
|
|
14
|
+
if (documents instanceof ReadableStream) {
|
|
15
|
+
body = documents;
|
|
17
16
|
}
|
|
18
17
|
else if (typeof documents === 'string') {
|
|
19
18
|
if (format === 'csv') {
|
|
@@ -25,11 +24,11 @@ async function importDocuments(ctx, documents, format = 'jsonl', importOptions,
|
|
|
25
24
|
const jsonlData = parsedDocuments
|
|
26
25
|
.map((doc) => JSON.stringify(doc))
|
|
27
26
|
.join('\n');
|
|
28
|
-
|
|
27
|
+
body = jsonlData;
|
|
29
28
|
}
|
|
30
29
|
else {
|
|
31
30
|
// Assume it's already JSONL
|
|
32
|
-
|
|
31
|
+
body = documents;
|
|
33
32
|
}
|
|
34
33
|
}
|
|
35
34
|
else {
|
|
@@ -38,16 +37,16 @@ async function importDocuments(ctx, documents, format = 'jsonl', importOptions,
|
|
|
38
37
|
throw new typesense_model_1.TypesenseError('CSV import requires conversion', 400);
|
|
39
38
|
}
|
|
40
39
|
const formatted = export_formatter_1.ExportFormatter.formatDocuments(documents, format);
|
|
41
|
-
|
|
40
|
+
body = formatted;
|
|
42
41
|
}
|
|
43
42
|
const searchParams = {
|
|
44
43
|
...importOptions,
|
|
45
44
|
action: importOptions?.action || 'create',
|
|
46
45
|
};
|
|
47
|
-
//
|
|
46
|
+
// Send body directly to HTTP for large files
|
|
48
47
|
const response = await ctx.httpClient.requestTextWithRawBody(`/collections/${collectionName}/documents/import`, {
|
|
49
48
|
method: 'POST',
|
|
50
|
-
body
|
|
49
|
+
body,
|
|
51
50
|
searchParams,
|
|
52
51
|
timeout: ctx.httpClient.importTimeout,
|
|
53
52
|
});
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"importDocuments.js","sourceRoot":"","sources":["../../../src/actions/documents/importDocuments.ts"],"names":[],"mappings":";;
|
|
1
|
+
{"version":3,"file":"importDocuments.js","sourceRoot":"","sources":["../../../src/actions/documents/importDocuments.ts"],"names":[],"mappings":";;AAWA,0CA+DC;AA1ED,wEAAmE;AASnE,2DAAsD;AAE/C,KAAK,UAAU,eAAe,CACnC,GAAqB,EACrB,SAA2D,EAC3D,SAAgC,OAAO,EACvC,aAAsC,EACtC,iBAA8C;IAE9C,kBAAkB;IAClB,MAAM,gBAAgB,GAA4B,CAAC,OAAO,EAAE,MAAM,EAAE,KAAK,CAAC,CAAA;IAC1E,IAAI,CAAC,gBAAgB,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;QACvC,MAAM,IAAI,gCAAc,CAAC,oBAAoB,EAAE,GAAG,CAAC,CAAA;IACrD,CAAC;IAED,MAAM,cAAc,GAAG,iBAAiB,EAAE,UAAU,IAAI,GAAG,CAAC,IAAI,EAAE,CAAA;IAClE,IAAI,IAA6B,CAAA;IAEjC,IAAI,SAAS,YAAY,cAAc,EAAE,CAAC;QACxC,IAAI,GAAG,SAAS,CAAA;IAClB,CAAC;SAAM,IAAI,OAAO,SAAS,KAAK,QAAQ,EAAE,CAAC;QACzC,IAAI,MAAM,KAAK,KAAK,EAAE,CAAC;YACrB,MAAM,IAAI,gCAAc,CAAC,gCAAgC,EAAE,GAAG,CAAC,CAAA;QACjE,CAAC;QACD,IAAI,MAAM,KAAK,MAAM,EAAE,CAAC;YACtB,qCAAqC;YACrC,MAAM,eAAe,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAA;YAC7C,MAAM,SAAS,GAAG,eAAe;iBAC9B,GAAG,CAAC,CAAC,GAAQ,EAAE,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;iBACtC,IAAI,CAAC,IAAI,CAAC,CAAA;YACb,IAAI,GAAG,SAAS,CAAA;QAClB,CAAC;aAAM,CAAC;YACN,4BAA4B;YAC5B,IAAI,GAAG,SAAS,CAAA;QAClB,CAAC;IACH,CAAC;SAAM,CAAC;QACN,qBAAqB;QACrB,IAAI,MAAM,KAAK,KAAK,EAAE,CAAC;YACrB,MAAM,IAAI,gCAAc,CAAC,gCAAgC,EAAE,GAAG,CAAC,CAAA;QACjE,CAAC;QACD,MAAM,SAAS,GAAG,kCAAe,CAAC,eAAe,CAAC,SAAS,EAAE,MAAM,CAAC,CAAA;QACpE,IAAI,GAAG,SAAmB,CAAA;IAC5B,CAAC;IAED,MAAM,YAAY,GAAQ;QACxB,GAAG,aAAa;QAChB,MAAM,EAAE,aAAa,EAAE,MAAM,IAAI,QAAQ;KAC1C,CAAA;IAED,6CAA6C;IAC7C,MAAM,QAAQ,GAAG,MAAM,GAAG,CAAC,UAAU,CAAC,sBAAsB,CAC1D,gBAAgB,cAAc,mBAAmB,EACjD;QACE,MAAM,EAAE,MAAM;QACd,IAAI;QACJ,YAAY;QACZ,OAAO,EAAE,GAAG,CAAC,UAAU,CAAC,aAAa;KACtC,CACF,CAAA;IAED,gCAAgC;IAChC,OAAO,QAAQ;SACZ,KAAK,CAAC,IAAI,CAAC;SACX,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;SAC3B,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAA;AAClC,CAAC"}
|
|
@@ -1,16 +1,15 @@
|
|
|
1
|
-
import { Readable, Transform } from 'node:stream';
|
|
2
1
|
import { TypesenseDocument, TypesenseExportFormat } from '../typesense.model';
|
|
3
2
|
export declare class ExportFormatter {
|
|
4
3
|
static formatDocuments<T>(documents: TypesenseDocument<T>[], format: TypesenseExportFormat): string | TypesenseDocument<T>[];
|
|
5
4
|
static formatCSV<T>(documents: TypesenseDocument<T>[]): string;
|
|
6
|
-
static createStreamingCSVTransform<T>():
|
|
7
|
-
static createStreamingJSONLTransform<T>():
|
|
8
|
-
static createGzipStream():
|
|
9
|
-
static createDocumentParser(format: TypesenseExportFormat):
|
|
5
|
+
static createStreamingCSVTransform<T>(): TransformStream<TypesenseDocument<T>, string>;
|
|
6
|
+
static createStreamingJSONLTransform<T>(): TransformStream<TypesenseDocument<T>, string>;
|
|
7
|
+
static createGzipStream(): never;
|
|
8
|
+
static createDocumentParser(format: TypesenseExportFormat): TransformStream<string, any>;
|
|
10
9
|
private static createJSONLParser;
|
|
11
10
|
private static createJSONParser;
|
|
12
11
|
private static escapeCsvValue;
|
|
13
|
-
static streamToString(stream:
|
|
14
|
-
static streamToAsyncIterator<T>(stream:
|
|
15
|
-
static createDocumentStream<T>(documents: TypesenseDocument<T>[]):
|
|
12
|
+
static streamToString(stream: ReadableStream): Promise<string>;
|
|
13
|
+
static streamToAsyncIterator<T>(stream: ReadableStream<T>): AsyncGenerator<T>;
|
|
14
|
+
static createDocumentStream<T>(documents: TypesenseDocument<T>[]): ReadableStream<T>;
|
|
16
15
|
}
|
|
@@ -1,9 +1,6 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.ExportFormatter = void 0;
|
|
4
|
-
// ExportFormatter - Handles CSV/gzip helpers with streaming support
|
|
5
|
-
const node_stream_1 = require("node:stream");
|
|
6
|
-
const node_zlib_1 = require("node:zlib");
|
|
7
4
|
class ExportFormatter {
|
|
8
5
|
static formatDocuments(documents, format) {
|
|
9
6
|
switch (format) {
|
|
@@ -40,14 +37,13 @@ class ExportFormatter {
|
|
|
40
37
|
static createStreamingCSVTransform() {
|
|
41
38
|
let isFirstRow = true;
|
|
42
39
|
let headers = [];
|
|
43
|
-
return new
|
|
44
|
-
|
|
45
|
-
transform(chunk, _encoding, callback) {
|
|
40
|
+
return new TransformStream({
|
|
41
|
+
transform(chunk, controller) {
|
|
46
42
|
try {
|
|
47
43
|
if (isFirstRow) {
|
|
48
44
|
// Extract headers from first document
|
|
49
45
|
headers = Object.keys(chunk).sort();
|
|
50
|
-
|
|
46
|
+
controller.enqueue(`${headers.join(',')}\n`);
|
|
51
47
|
isFirstRow = false;
|
|
52
48
|
}
|
|
53
49
|
// Convert document to CSV row
|
|
@@ -55,31 +51,23 @@ class ExportFormatter {
|
|
|
55
51
|
const value = chunk[header];
|
|
56
52
|
return ExportFormatter.escapeCsvValue(value);
|
|
57
53
|
});
|
|
58
|
-
|
|
59
|
-
callback();
|
|
54
|
+
controller.enqueue(`${row.join(',')}\n`);
|
|
60
55
|
}
|
|
61
56
|
catch (error) {
|
|
62
|
-
|
|
57
|
+
controller.error(error);
|
|
63
58
|
}
|
|
64
59
|
},
|
|
65
60
|
});
|
|
66
61
|
}
|
|
67
62
|
static createStreamingJSONLTransform() {
|
|
68
|
-
return new
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
try {
|
|
72
|
-
this.push(`${JSON.stringify(chunk)}\n`);
|
|
73
|
-
callback();
|
|
74
|
-
}
|
|
75
|
-
catch (error) {
|
|
76
|
-
callback(error);
|
|
77
|
-
}
|
|
63
|
+
return new TransformStream({
|
|
64
|
+
transform(chunk, controller) {
|
|
65
|
+
controller.enqueue(`${JSON.stringify(chunk)}\n`);
|
|
78
66
|
},
|
|
79
67
|
});
|
|
80
68
|
}
|
|
81
69
|
static createGzipStream() {
|
|
82
|
-
|
|
70
|
+
throw new Error('createGzipStream is not available in browser environments. Use CompressionStream API or a server-side solution.');
|
|
83
71
|
}
|
|
84
72
|
static createDocumentParser(format) {
|
|
85
73
|
switch (format) {
|
|
@@ -93,62 +81,55 @@ class ExportFormatter {
|
|
|
93
81
|
}
|
|
94
82
|
static createJSONLParser() {
|
|
95
83
|
let buffer = '';
|
|
96
|
-
return new
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
buffer += chunk.toString();
|
|
84
|
+
return new TransformStream({
|
|
85
|
+
transform(chunk, controller) {
|
|
86
|
+
buffer += chunk;
|
|
100
87
|
const lines = buffer.split('\n');
|
|
101
88
|
// Keep the last incomplete line in buffer
|
|
102
89
|
buffer = lines.pop() || '';
|
|
103
90
|
for (const line of lines) {
|
|
104
91
|
if (line.trim()) {
|
|
105
92
|
try {
|
|
106
|
-
|
|
107
|
-
this.push(document);
|
|
93
|
+
controller.enqueue(JSON.parse(line));
|
|
108
94
|
}
|
|
109
95
|
catch (_error) {
|
|
110
|
-
|
|
96
|
+
controller.error(new Error(`Invalid JSON in line: ${line}`));
|
|
97
|
+
return;
|
|
111
98
|
}
|
|
112
99
|
}
|
|
113
100
|
}
|
|
114
|
-
callback();
|
|
115
101
|
},
|
|
116
|
-
flush(
|
|
102
|
+
flush(controller) {
|
|
117
103
|
if (buffer.trim()) {
|
|
118
104
|
try {
|
|
119
|
-
|
|
120
|
-
this.push(document);
|
|
105
|
+
controller.enqueue(JSON.parse(buffer));
|
|
121
106
|
}
|
|
122
107
|
catch (_error) {
|
|
123
|
-
|
|
108
|
+
controller.error(new Error(`Invalid JSON in final line: ${buffer}`));
|
|
124
109
|
}
|
|
125
110
|
}
|
|
126
|
-
callback();
|
|
127
111
|
},
|
|
128
112
|
});
|
|
129
113
|
}
|
|
130
114
|
static createJSONParser() {
|
|
131
115
|
let buffer = '';
|
|
132
|
-
return new
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
buffer += chunk.toString();
|
|
136
|
-
callback();
|
|
116
|
+
return new TransformStream({
|
|
117
|
+
transform(chunk, _controller) {
|
|
118
|
+
buffer += chunk;
|
|
137
119
|
},
|
|
138
|
-
flush(
|
|
120
|
+
flush(controller) {
|
|
139
121
|
try {
|
|
140
122
|
const documents = JSON.parse(buffer);
|
|
141
123
|
if (Array.isArray(documents)) {
|
|
142
|
-
documents.forEach(doc =>
|
|
124
|
+
documents.forEach(doc => controller.enqueue(doc));
|
|
143
125
|
}
|
|
144
126
|
else {
|
|
145
|
-
|
|
127
|
+
controller.enqueue(documents);
|
|
146
128
|
}
|
|
147
129
|
}
|
|
148
130
|
catch (error) {
|
|
149
|
-
|
|
131
|
+
controller.error(new Error(`Invalid JSON: ${error.message}`));
|
|
150
132
|
}
|
|
151
|
-
callback();
|
|
152
133
|
},
|
|
153
134
|
});
|
|
154
135
|
}
|
|
@@ -178,45 +159,46 @@ class ExportFormatter {
|
|
|
178
159
|
return stringValue;
|
|
179
160
|
}
|
|
180
161
|
static async streamToString(stream) {
|
|
181
|
-
const
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
162
|
+
const reader = stream.getReader();
|
|
163
|
+
const decoder = new TextDecoder();
|
|
164
|
+
let result = '';
|
|
165
|
+
while (true) {
|
|
166
|
+
const { done, value } = await reader.read();
|
|
167
|
+
if (done) {
|
|
168
|
+
break;
|
|
169
|
+
}
|
|
170
|
+
result +=
|
|
171
|
+
typeof value === 'string'
|
|
172
|
+
? value
|
|
173
|
+
: decoder.decode(value, { stream: true });
|
|
174
|
+
}
|
|
175
|
+
result += decoder.decode();
|
|
176
|
+
return result;
|
|
189
177
|
}
|
|
190
178
|
static async *streamToAsyncIterator(stream) {
|
|
191
|
-
const reader = stream
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
179
|
+
const reader = stream.getReader();
|
|
180
|
+
try {
|
|
181
|
+
while (true) {
|
|
182
|
+
const { done, value } = await reader.read();
|
|
183
|
+
if (done) {
|
|
184
|
+
break;
|
|
185
|
+
}
|
|
186
|
+
yield value;
|
|
195
187
|
}
|
|
196
188
|
}
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
const chunks = [];
|
|
200
|
-
await new Promise((resolve, reject) => {
|
|
201
|
-
stream.on('data', (chunk) => chunks.push(chunk));
|
|
202
|
-
stream.on('error', reject);
|
|
203
|
-
stream.on('end', resolve);
|
|
204
|
-
});
|
|
205
|
-
for (const chunk of chunks) {
|
|
206
|
-
yield chunk;
|
|
207
|
-
}
|
|
189
|
+
finally {
|
|
190
|
+
reader.releaseLock();
|
|
208
191
|
}
|
|
209
192
|
}
|
|
210
193
|
static createDocumentStream(documents) {
|
|
211
194
|
let index = 0;
|
|
212
|
-
return new
|
|
213
|
-
|
|
214
|
-
read() {
|
|
195
|
+
return new ReadableStream({
|
|
196
|
+
pull(controller) {
|
|
215
197
|
if (index < documents.length) {
|
|
216
|
-
|
|
198
|
+
controller.enqueue(documents[index++]);
|
|
217
199
|
}
|
|
218
200
|
else {
|
|
219
|
-
|
|
201
|
+
controller.close();
|
|
220
202
|
}
|
|
221
203
|
},
|
|
222
204
|
});
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"export-formatter.js","sourceRoot":"","sources":["../../src/components/export-formatter.ts"],"names":[],"mappings":";;;
|
|
1
|
+
{"version":3,"file":"export-formatter.js","sourceRoot":"","sources":["../../src/components/export-formatter.ts"],"names":[],"mappings":";;;AAGA,MAAa,eAAe;IAC1B,MAAM,CAAC,eAAe,CACpB,SAAiC,EACjC,MAA6B;QAE7B,QAAQ,MAAM,EAAE,CAAC;YACf,KAAK,MAAM;gBACT,OAAO,SAAS,CAAA;YAClB,KAAK,OAAO;gBACV,OAAO,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;YAC7D,KAAK,KAAK;gBACR,OAAO,eAAe,CAAC,SAAS,CAAC,SAAS,CAAC,CAAA;YAC7C;gBACE,MAAM,IAAI,KAAK,CAAC,8BAA8B,MAAM,EAAE,CAAC,CAAA;QAC3D,CAAC;IACH,CAAC;IAED,MAAM,CAAC,SAAS,CAAI,SAAiC;QACnD,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC3B,OAAO,EAAE,CAAA;QACX,CAAC;QAED,yCAAyC;QACzC,MAAM,OAAO,GAAG,IAAI,GAAG,EAAU,CAAA;QACjC,SAAS,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE;YACtB,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAA;QACnD,CAAC,CAAC,CAAA;QAEF,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,CAAA;QAC1C,MAAM,QAAQ,GAAa,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAA;QAE9C,KAAK,MAAM,GAAG,IAAI,SAAS,EAAE,CAAC;YAC5B,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE;gBAC/B,MAAM,KAAK,GAAI,GAAW,CAAC,MAAM,CAAC,CAAA;gBAClC,OAAO,eAAe,CAAC,cAAc,CAAC,KAAK,CAAC,CAAA;YAC9C,CAAC,CAAC,CAAA;YACF,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAA;QAC9B,CAAC;QAED,OAAO,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;IAC5B,CAAC;IAED,MAAM,CAAC,2BAA2B;QAIhC,IAAI,UAAU,GAAG,IAAI,CAAA;QACrB,IAAI,OAAO,GAAa,EAAE,CAAA;QAE1B,OAAO,IAAI,eAAe,CAAC;YACzB,SAAS,CAAC,KAAK,EAAE,UAAU;gBACzB,IAAI,CAAC;oBACH,IAAI,UAAU,EAAE,CAAC;wBACf,sCAAsC;wBACtC,OAAO,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,EAAE,CAAA;wBACnC,UAAU,CAAC,OAAO,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;wBAC5C,UAAU,GAAG,KAAK,CAAA;oBACpB,CAAC;oBAED,8BAA8B;oBAC9B,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE;wBAC/B,MAAM,KAAK,GAAI,KAAa,CAAC,MAAM,CAAC,CAAA;wBACpC,OAAO,eAAe,CAAC,cAAc,CAAC,KAAK,CAAC,CAAA;oBAC9C,CAAC,CAAC,CAAA;oBAEF,UAAU,CAAC,OAAO,CAAC,GAAG,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;gBAC1C,CAAC;gBAAC,OAAO,KAAU,EAAE,CAAC;oBACpB,UAAU,CAAC,KAAK,CAAC,KAAK,CAAC,CAAA;gBACzB,CAAC;YACH,CAAC;SACF,CAAC,CAAA;IACJ,CAAC;IAED,MAAM,CAAC,6BAA6B;QAIlC,OAAO,IAAI,eAAe,CAAC;YACzB,SAAS,CAAC,KAAK,EAAE,UAAU;gBACzB,UAAU,CAAC,OAAO,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,CAAC,CAAA;YAClD,CAAC;SACF,CAAC,CAAA;IACJ,CAAC;IAED,MAAM,CAAC,gBAAgB;QACrB,MAAM,IAAI,KAAK,CACb,iHAAiH,CAClH,CAAA;IACH,CAAC;IAED,MAAM,CAAC,oBAAoB,CACzB,MAA6B;QAE7B,QAAQ,MAAM,EAAE,CAAC;YACf,KAAK,OAAO;gBACV,OAAO,eAAe,CAAC,iBAAiB,EAAE,CAAA;YAC5C,KAAK,MAAM;gBACT,OAAO,eAAe,CAAC,gBAAgB,EAAE,CAAA;YAC3C;gBACE,MAAM,IAAI,KAAK,CAAC,qCAAqC,MAAM,EAAE,CAAC,CAAA;QAClE,CAAC;IACH,CAAC;IAEO,MAAM,CAAC,iBAAiB;QAC9B,IAAI,MAAM,GAAG,EAAE,CAAA;QAEf,OAAO,IAAI,eAAe,CAAC;YACzB,SAAS,CAAC,KAAK,EAAE,UAAU;gBACzB,MAAM,IAAI,KAAK,CAAA;gBACf,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAA;gBAEhC,0CAA0C;gBAC1C,MAAM,GAAG,KAAK,CAAC,GAAG,EAAE,IAAI,EAAE,CAAA;gBAE1B,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;oBACzB,IAAI,IAAI,CAAC,IAAI,EAAE,EAAE,CAAC;wBAChB,IAAI,CAAC;4BACH,UAAU,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAA;wBACtC,CAAC;wBAAC,OAAO,MAAM,EAAE,CAAC;4BAChB,UAAU,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,yBAAyB,IAAI,EAAE,CAAC,CAAC,CAAA;4BAC5D,OAAM;wBACR,CAAC;oBACH,CAAC;gBACH,CAAC;YACH,CAAC;YAED,KAAK,CAAC,UAAU;gBACd,IAAI,MAAM,CAAC,IAAI,EAAE,EAAE,CAAC;oBAClB,IAAI,CAAC;wBACH,UAAU,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAA;oBACxC,CAAC;oBAAC,OAAO,MAAM,EAAE,CAAC;wBAChB,UAAU,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,+BAA+B,MAAM,EAAE,CAAC,CAAC,CAAA;oBACtE,CAAC;gBACH,CAAC;YACH,CAAC;SACF,CAAC,CAAA;IACJ,CAAC;IAEO,MAAM,CAAC,gBAAgB;QAC7B,IAAI,MAAM,GAAG,EAAE,CAAA;QAEf,OAAO,IAAI,eAAe,CAAC;YACzB,SAAS,CAAC,KAAK,EAAE,WAAW;gBAC1B,MAAM,IAAI,KAAK,CAAA;YACjB,CAAC;YAED,KAAK,CAAC,UAAU;gBACd,IAAI,CAAC;oBACH,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAA;oBACpC,IAAI,KAAK,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,CAAC;wBAC7B,SAAS,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAA;oBACnD,CAAC;yBAAM,CAAC;wBACN,UAAU,CAAC,OAAO,CAAC,SAAS,CAAC,CAAA;oBAC/B,CAAC;gBACH,CAAC;gBAAC,OAAO,KAAU,EAAE,CAAC;oBACpB,UAAU,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,iBAAiB,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC,CAAA;gBAC/D,CAAC;YACH,CAAC;SACF,CAAC,CAAA;IACJ,CAAC;IAEO,MAAM,CAAC,cAAc,CAAC,KAAU;QACtC,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;YAC1C,OAAO,EAAE,CAAA;QACX,CAAC;QAED,IAAI,WAAW,GAAG,MAAM,CAAC,KAAK,CAAC,CAAA;QAE/B,2CAA2C;QAC3C,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;YACzB,WAAW,GAAG,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;QACzD,CAAC;QAED,iCAAiC;QACjC,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;YACvD,WAAW,GAAG,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAA;QACrC,CAAC;QAED,6CAA6C;QAC7C,MAAM,YAAY,GAChB,WAAW,CAAC,QAAQ,CAAC,GAAG,CAAC;YACzB,WAAW,CAAC,QAAQ,CAAC,GAAG,CAAC;YACzB,WAAW,CAAC,QAAQ,CAAC,IAAI,CAAC;YAC1B,WAAW,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAA;QAE5B,IAAI,YAAY,EAAE,CAAC;YACjB,0CAA0C;YAC1C,WAAW,GAAG,WAAW,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC,CAAA;YAC7C,OAAO,IAAI,WAAW,GAAG,CAAA;QAC3B,CAAC;QAED,OAAO,WAAW,CAAA;IACpB,CAAC;IAED,MAAM,CAAC,KAAK,CAAC,cAAc,CAAC,MAAsB;QAChD,MAAM,MAAM,GAAG,MAAM,CAAC,SAAS,EAAE,CAAA;QACjC,MAAM,OAAO,GAAG,IAAI,WAAW,EAAE,CAAA;QACjC,IAAI,MAAM,GAAG,EAAE,CAAA;QACf,OAAO,IAAI,EAAE,CAAC;YACZ,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,MAAM,MAAM,CAAC,IAAI,EAAE,CAAA;YAC3C,IAAI,IAAI,EAAE,CAAC;gBACT,MAAK;YACP,CAAC;YACD,MAAM;gBACJ,OAAO,KAAK,KAAK,QAAQ;oBACvB,CAAC,CAAC,KAAK;oBACP,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAA;QAC/C,CAAC;QACD,MAAM,IAAI,OAAO,CAAC,MAAM,EAAE,CAAA;QAC1B,OAAO,MAAM,CAAA;IACf,CAAC;IAED,MAAM,CAAC,KAAK,CAAC,CAAC,qBAAqB,CACjC,MAAyB;QAEzB,MAAM,MAAM,GAAG,MAAM,CAAC,SAAS,EAAE,CAAA;QACjC,IAAI,CAAC;YACH,OAAO,IAAI,EAAE,CAAC;gBACZ,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,MAAM,MAAM,CAAC,IAAI,EAAE,CAAA;gBAC3C,IAAI,IAAI,EAAE,CAAC;oBACT,MAAK;gBACP,CAAC;gBACD,MAAM,KAAK,CAAA;YACb,CAAC;QACH,CAAC;gBAAS,CAAC;YACT,MAAM,CAAC,WAAW,EAAE,CAAA;QACtB,CAAC;IACH,CAAC;IAED,MAAM,CAAC,oBAAoB,CACzB,SAAiC;QAEjC,IAAI,KAAK,GAAG,CAAC,CAAA;QAEb,OAAO,IAAI,cAAc,CAAC;YACxB,IAAI,CAAC,UAAU;gBACb,IAAI,KAAK,GAAG,SAAS,CAAC,MAAM,EAAE,CAAC;oBAC7B,UAAU,CAAC,OAAO,CAAC,SAAS,CAAC,KAAK,EAAE,CAAM,CAAC,CAAA;gBAC7C,CAAC;qBAAM,CAAC;oBACN,UAAU,CAAC,KAAK,EAAE,CAAA;gBACpB,CAAC;YACH,CAAC;SACF,CAAC,CAAA;IACJ,CAAC;CACF;AApPD,0CAoPC"}
|
package/package.json
CHANGED