@dpuse/dpuse-shared 1.0.214 → 1.0.215

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,5 +1,6 @@
1
1
  import { FileTypeResult } from 'file-type';
2
2
  import { ComponentInstanceConfig } from '..';
3
+ import { StructurePath } from '../module/tool';
3
4
  import { ConnectionNodeConfig, ObjectColumnConfig } from '../connection';
4
5
  export interface DataViewConfig extends ComponentInstanceConfig {
5
6
  typeId: 'dataView';
@@ -23,6 +24,7 @@ export interface ReadingConfig {
23
24
  isDeclared?: boolean | undefined;
24
25
  parserId: string | undefined;
25
26
  recordDelimiterId: RecordDelimiterId | undefined;
27
+ structurePath?: StructurePath | undefined;
26
28
  valueDelimiterId: ValueDelimiterId | undefined;
27
29
  }
28
30
  export type DataFormatId = (typeof DATA_FORMAT_CONFIGS)[number]['id'];
@@ -31,6 +33,7 @@ export interface ReadingChoiceConfig {
31
33
  encodingId: string | undefined;
32
34
  hasHeaders: boolean | undefined;
33
35
  parserId: string | undefined;
36
+ structurePath?: StructurePath | undefined;
34
37
  valueDelimiterId: ValueDelimiterId | undefined;
35
38
  }
36
39
  export type DataFormatSourceId = 'fileContent' | 'jsonSyntax' | 'mimeType' | 'extension' | 'default';
@@ -47,10 +50,12 @@ export interface SampleConfig {
47
50
  recordDelimiterId?: RecordDelimiterId | undefined;
48
51
  replacementCharacterCount?: number | undefined;
49
52
  size: number | undefined;
53
+ structurePath?: StructurePath | undefined;
50
54
  text: string | undefined;
51
55
  valueDelimiterId?: ValueDelimiterId | undefined;
52
56
  }
53
57
  export interface SampleIssue {
58
+ columnNumber?: number | undefined;
54
59
  endLineNumber: number | undefined;
55
60
  fieldNumber: number | undefined;
56
61
  id: string;
@@ -171,66 +176,79 @@ export type TemporalSubtypeId = Extract<DataTypeConfig, {
171
176
  }>['subtypeIds'][number];
172
177
  export declare const DATA_FORMAT_CONFIGS: readonly [{
173
178
  readonly id: "arrow";
179
+ readonly isStructured: false;
174
180
  readonly isTable: false;
175
181
  readonly isText: false;
176
182
  readonly supportId: "planned";
177
183
  }, {
178
184
  readonly id: "avro";
185
+ readonly isStructured: false;
179
186
  readonly isTable: false;
180
187
  readonly isText: false;
181
188
  readonly supportId: "planned";
182
189
  }, {
183
190
  readonly id: "dtv";
191
+ readonly isStructured: false;
184
192
  readonly isTable: true;
185
193
  readonly isText: true;
186
194
  readonly supportId: "supported";
187
195
  }, {
188
196
  readonly id: "json";
189
- readonly isTable: false;
190
- readonly isText: false;
191
- readonly supportId: "planned";
197
+ readonly isStructured: true;
198
+ readonly isTable: true;
199
+ readonly isText: true;
200
+ readonly supportId: "supported";
192
201
  }, {
193
202
  readonly id: "jsonl";
194
- readonly isTable: false;
195
- readonly isText: false;
196
- readonly supportId: "planned";
203
+ readonly isStructured: true;
204
+ readonly isTable: true;
205
+ readonly isText: true;
206
+ readonly supportId: "supported";
197
207
  }, {
198
208
  readonly id: "markdown";
209
+ readonly isStructured: false;
199
210
  readonly isTable: false;
200
211
  readonly isText: true;
201
212
  readonly supportId: "unsupported";
202
213
  }, {
203
214
  readonly id: "ods";
215
+ readonly isStructured: true;
204
216
  readonly isTable: false;
205
217
  readonly isText: false;
206
218
  readonly supportId: "planned";
207
219
  }, {
208
220
  readonly id: "parquet";
221
+ readonly isStructured: false;
209
222
  readonly isTable: false;
210
223
  readonly isText: false;
211
224
  readonly supportId: "planned";
212
225
  }, {
213
226
  readonly id: "sav";
227
+ readonly isStructured: false;
214
228
  readonly isTable: false;
215
229
  readonly isText: false;
216
230
  readonly supportId: "planned";
217
231
  }, {
218
232
  readonly id: "sqlite";
233
+ readonly isStructured: false;
219
234
  readonly isTable: false;
220
235
  readonly isText: false;
221
236
  readonly supportId: "planned";
222
237
  }, {
223
238
  readonly id: "xlsx";
239
+ readonly isStructured: true;
224
240
  readonly isTable: false;
225
241
  readonly isText: false;
226
242
  readonly supportId: "planned";
227
243
  }, {
228
244
  readonly id: "xml";
245
+ readonly isStructured: true;
229
246
  readonly isTable: false;
230
247
  readonly isText: false;
231
248
  readonly supportId: "planned";
232
249
  }, {
233
250
  readonly id: "unknown";
251
+ readonly isStructured: false;
234
252
  readonly isTable: false;
235
253
  readonly isText: false;
236
254
  readonly supportId: "unsupported";
@@ -2,7 +2,7 @@
2
2
  * Defines the configuration metadata for a connector. Used for validation
3
3
  * of connector manifests and capability discovery at runtime.
4
4
  */
5
- export declare const connectorActionNameSchema: import('valibot').UnionSchema<readonly [import('valibot').LiteralSchema<"auditObjectContent", undefined>, import('valibot').LiteralSchema<"closeStore", undefined>, import('valibot').LiteralSchema<"createObject", undefined>, import('valibot').LiteralSchema<"describeConnection", undefined>, import('valibot').LiteralSchema<"dropObject", undefined>, import('valibot').LiteralSchema<"findObject", undefined>, import('valibot').LiteralSchema<"getInfo", undefined>, import('valibot').LiteralSchema<"getReadableStream", undefined>, import('valibot').LiteralSchema<"getRecord", undefined>, import('valibot').LiteralSchema<"listNodes", undefined>, import('valibot').LiteralSchema<"loadObjectRecords", undefined>, import('valibot').LiteralSchema<"openStore", undefined>, import('valibot').LiteralSchema<"previewObject", undefined>, import('valibot').LiteralSchema<"queryRecords", undefined>, import('valibot').LiteralSchema<"removeRecords", undefined>, import('valibot').LiteralSchema<"retrieveRecords", undefined>, import('valibot').LiteralSchema<"upsertRecords", undefined>], undefined>;
5
+ export declare const connectorActionNameSchema: import('valibot').UnionSchema<readonly [import('valibot').LiteralSchema<"auditObjectContent", undefined>, import('valibot').LiteralSchema<"closeStore", undefined>, import('valibot').LiteralSchema<"createObject", undefined>, import('valibot').LiteralSchema<"describeConnection", undefined>, import('valibot').LiteralSchema<"dropObject", undefined>, import('valibot').LiteralSchema<"findObject", undefined>, import('valibot').LiteralSchema<"getInfo", undefined>, import('valibot').LiteralSchema<"getReadableStream", undefined>, import('valibot').LiteralSchema<"getRecord", undefined>, import('valibot').LiteralSchema<"indexObjectStructure", undefined>, import('valibot').LiteralSchema<"listNodes", undefined>, import('valibot').LiteralSchema<"listObjectStructureEntries", undefined>, import('valibot').LiteralSchema<"loadObjectRecords", undefined>, import('valibot').LiteralSchema<"openStore", undefined>, import('valibot').LiteralSchema<"previewObject", undefined>, import('valibot').LiteralSchema<"queryRecords", undefined>, import('valibot').LiteralSchema<"removeRecords", undefined>, import('valibot').LiteralSchema<"retrieveRecords", undefined>, import('valibot').LiteralSchema<"upsertRecords", undefined>], undefined>;
6
6
  export declare const connectorCategoryConfigSchema: import('valibot').StrictObjectSchema<{
7
7
  readonly label: import('valibot').StrictObjectSchema<{
8
8
  readonly en: import('valibot').StringSchema<undefined>;
@@ -25,7 +25,7 @@ export declare const connectorUsageConfigSchema: import('valibot').StrictObjectS
25
25
  }, undefined>;
26
26
  export declare const connectorConfigSchema: import('valibot').StrictObjectSchema<{
27
27
  readonly typeId: import('valibot').LiteralSchema<"connector", undefined>;
28
- readonly actionNames: import('valibot').ArraySchema<import('valibot').UnionSchema<readonly [import('valibot').LiteralSchema<"auditObjectContent", undefined>, import('valibot').LiteralSchema<"closeStore", undefined>, import('valibot').LiteralSchema<"createObject", undefined>, import('valibot').LiteralSchema<"describeConnection", undefined>, import('valibot').LiteralSchema<"dropObject", undefined>, import('valibot').LiteralSchema<"findObject", undefined>, import('valibot').LiteralSchema<"getInfo", undefined>, import('valibot').LiteralSchema<"getReadableStream", undefined>, import('valibot').LiteralSchema<"getRecord", undefined>, import('valibot').LiteralSchema<"listNodes", undefined>, import('valibot').LiteralSchema<"loadObjectRecords", undefined>, import('valibot').LiteralSchema<"openStore", undefined>, import('valibot').LiteralSchema<"previewObject", undefined>, import('valibot').LiteralSchema<"queryRecords", undefined>, import('valibot').LiteralSchema<"removeRecords", undefined>, import('valibot').LiteralSchema<"retrieveRecords", undefined>, import('valibot').LiteralSchema<"upsertRecords", undefined>], undefined>, undefined>;
28
+ readonly actionNames: import('valibot').ArraySchema<import('valibot').UnionSchema<readonly [import('valibot').LiteralSchema<"auditObjectContent", undefined>, import('valibot').LiteralSchema<"closeStore", undefined>, import('valibot').LiteralSchema<"createObject", undefined>, import('valibot').LiteralSchema<"describeConnection", undefined>, import('valibot').LiteralSchema<"dropObject", undefined>, import('valibot').LiteralSchema<"findObject", undefined>, import('valibot').LiteralSchema<"getInfo", undefined>, import('valibot').LiteralSchema<"getReadableStream", undefined>, import('valibot').LiteralSchema<"getRecord", undefined>, import('valibot').LiteralSchema<"indexObjectStructure", undefined>, import('valibot').LiteralSchema<"listNodes", undefined>, import('valibot').LiteralSchema<"listObjectStructureEntries", undefined>, import('valibot').LiteralSchema<"loadObjectRecords", undefined>, import('valibot').LiteralSchema<"openStore", undefined>, import('valibot').LiteralSchema<"previewObject", undefined>, import('valibot').LiteralSchema<"queryRecords", undefined>, import('valibot').LiteralSchema<"removeRecords", undefined>, import('valibot').LiteralSchema<"retrieveRecords", undefined>, import('valibot').LiteralSchema<"upsertRecords", undefined>], undefined>, undefined>;
29
29
  readonly category: import('valibot').NullableSchema<import('valibot').StrictObjectSchema<{
30
30
  readonly label: import('valibot').StrictObjectSchema<{
31
31
  readonly en: import('valibot').StringSchema<undefined>;
@@ -2,7 +2,7 @@ import { InferOutput } from 'valibot';
2
2
  import { EncodingDetectionConfig } from '../../../encoding';
3
3
  import { EngineConnectorActionOptions } from '../engine';
4
4
  import { LocalisedConfig } from '../../../locale';
5
- import { ToolConfig } from '../tool';
5
+ import { ToolConfig, ListStructureEntriesResult, StructureIndex, StructureLocator, StructurePath } from '../tool';
6
6
  import { ColumnAuditConfig, ColumnReadingConfig, ContentAuditConfig, DataFormatId, DeclaredColumnConfig, ParsedRecord, ReadingConfig, SampleConfig, ValueDelimiterId } from '../../dataView';
7
7
  import { ConnectionConfig, ConnectionDescriptionConfig, ConnectionNodeConfig, ObjectColumnConfig } from '../../connection';
8
8
  import { connectorActionNameSchema, connectorCategoryConfigSchema, connectorConfigSchema, connectorItemTypeConfigSchema, connectorItemTypeIdSchema, connectorUsageConfigSchema, connectorUsageIdSchema } from './connectorConfig.schema';
@@ -17,7 +17,9 @@ export interface ConnectorInterface {
17
17
  getInfo?(options: GetInfoOptions): Promise<GetInfoResult>;
18
18
  getReadableStream?(options: GetReadableStreamOptions): Promise<ReadableStream<Uint8Array>>;
19
19
  getRecord?(options: GetRecordOptions): Promise<GetRecordResult>;
20
+ indexObjectStructure?(options: IndexObjectStructureOptions, chunk: (byteCount: number) => void): Promise<StructureIndex>;
20
21
  listNodes?(options: ListNodesOptions): Promise<ListNodesResult>;
22
+ listObjectStructureEntries?(options: ListObjectStructureEntriesOptions): Promise<ListStructureEntriesResult>;
21
23
  loadObjectRecords?(options: LoadObjectRecordsOptions, chunks: AsyncIterable<StoreRecord[]>): Promise<LoadObjectRecordsResult>;
22
24
  openStore?(options: OpenStoreOptions): Promise<OpenStoreResult>;
23
25
  previewObject?(options: PreviewObjectOptions): Promise<PreviewObjectResult>;
@@ -55,6 +57,7 @@ export interface AuditObjectContentOptions extends EngineConnectorActionOptions
55
57
  parserId: string | undefined;
56
58
  path: string;
57
59
  stopSignal: AbortSignal | undefined;
60
+ structurePath?: StructurePath | undefined;
58
61
  supportsTransferableStreams: boolean;
59
62
  valueDelimiterId: ValueDelimiterId | undefined;
60
63
  }
@@ -77,8 +80,7 @@ export interface DropObjectOptions extends EngineConnectorActionOptions {
77
80
  path: string;
78
81
  }
79
82
  export interface FindObjectOptions extends EngineConnectorActionOptions {
80
- storeId: string | undefined;
81
- nodeId: string;
83
+ id: string;
82
84
  }
83
85
  export interface FindObjectResult {
84
86
  path: string | undefined;
@@ -100,6 +102,12 @@ export type RecordKey = number | string;
100
102
  export interface GetRecordResult {
101
103
  record: StoreRecord;
102
104
  }
105
+ export interface IndexObjectStructureOptions extends EngineConnectorActionOptions {
106
+ dataFormatId: DataFormatId;
107
+ encodingId: string | undefined;
108
+ parserId: string | undefined;
109
+ path: string;
110
+ }
103
111
  export interface ListNodesOptions extends EngineConnectorActionOptions {
104
112
  folderPath: string;
105
113
  limit?: number;
@@ -112,6 +120,15 @@ export interface ListNodesResult {
112
120
  isMore: boolean;
113
121
  totalCount: number;
114
122
  }
123
+ export interface ListObjectStructureEntriesOptions extends EngineConnectorActionOptions {
124
+ dataFormatId: DataFormatId;
125
+ entryCount: number;
126
+ locator: StructureLocator;
127
+ maxValueLength: number | undefined;
128
+ parserId: string | undefined;
129
+ path: string;
130
+ structurePath: StructurePath;
131
+ }
115
132
  export interface LoadObjectRecordsOptions extends EngineConnectorActionOptions {
116
133
  columnConfigs: ObjectColumnConfig[];
117
134
  path: string;
@@ -146,6 +163,7 @@ export interface PreviewObjectOptions extends EngineConnectorActionOptions {
146
163
  mimeType: string | undefined;
147
164
  parserId: string | undefined;
148
165
  path: string;
166
+ structurePath?: StructurePath | undefined;
149
167
  valueDelimiterId: ValueDelimiterId | undefined;
150
168
  }
151
169
  export interface PreviewObjectResult {
@@ -153,6 +171,7 @@ export interface PreviewObjectResult {
153
171
  sampleConfig: SampleConfig;
154
172
  readingConfig: ReadingConfig;
155
173
  encodingCandidates?: EncodingDetectionConfig[] | undefined;
174
+ structureIndex?: StructureIndex | undefined;
156
175
  }
157
176
  export interface QueryRecordsOptions extends EngineConnectorActionOptions {
158
177
  aggregates: QueryAggregate[];
@@ -202,6 +221,7 @@ export interface RetrieveRecordsOptions extends EngineConnectorActionOptions {
202
221
  encodingId: string | undefined;
203
222
  parserId: string | undefined;
204
223
  path: string;
224
+ structurePath?: StructurePath | undefined;
205
225
  valueDelimiterId: ValueDelimiterId | undefined;
206
226
  }
207
227
  export interface RetrieveRecordsSummary {
@@ -221,7 +241,8 @@ export interface UpsertRecordsOptions extends EngineConnectorActionOptions {
221
241
  path: string;
222
242
  }
223
243
  export type ConnectorUsageId = InferOutput<typeof connectorUsageIdSchema>;
224
- export { auditObjectContentWithParser, buildPreviewObjectResult, retrieveRecordsWithParser } from './objectParsing';
244
+ export { auditObjectContentWithParser, buildPreviewObjectResult, indexObjectStructureWithParser } from './objectParsing';
245
+ export { listObjectStructureEntriesWithParser, retrieveRecordsWithParser } from './objectParsing';
225
246
  export { resolveStorePath } from './storePath';
226
247
  export { convertStoreRecordToParsedRecord, convertStoreValueToText } from './storeValues';
227
248
  export { acquireWebLock } from './webLock';
@@ -1,6 +1,6 @@
1
- import { AuditObjectContentOptions, AuditObjectContentResult, ConnectorUtilities, ParsedRecordsChunk, PreviewObjectResult, RetrieveRecordsOptions, RetrieveRecordsSummary } from '.';
1
+ import { RetrieveRecordsSummary, AuditObjectContentOptions, AuditObjectContentResult, ConnectorUtilities, IndexObjectStructureOptions, ListObjectStructureEntriesOptions, ParsedRecordsChunk, PreviewObjectResult, RetrieveRecordsOptions } from '.';
2
2
  import { DeclaredColumnConfig, DeclaredReadingConfig, ReadingChoiceConfig } from '../../dataView';
3
- import { FilePreviewResult, ToolConfig } from '../tool';
3
+ import { FilePreviewResult, ListStructureEntriesResult, StructureIndex, ToolConfig } from '../tool';
4
4
  export declare function buildPreviewObjectResult(input: {
5
5
  asAt: number;
6
6
  declaredColumnConfigs?: DeclaredColumnConfig[] | undefined;
@@ -26,3 +26,16 @@ export declare function auditObjectContentWithParser(input: {
26
26
  stream: ReadableStream<Uint8Array>;
27
27
  toolConfigs: ToolConfig[];
28
28
  }): Promise<AuditObjectContentResult>;
29
+ export declare function indexObjectStructureWithParser(input: {
30
+ onProgress: (byteCount: number) => void;
31
+ options: IndexObjectStructureOptions;
32
+ signal: AbortSignal;
33
+ stream: ReadableStream<Uint8Array>;
34
+ toolConfigs: ToolConfig[];
35
+ }): Promise<StructureIndex>;
36
+ export declare function listObjectStructureEntriesWithParser(input: {
37
+ options: ListObjectStructureEntriesOptions;
38
+ signal: AbortSignal;
39
+ stream: ReadableStream<Uint8Array>;
40
+ toolConfigs: ToolConfig[];
41
+ }): Promise<ListStructureEntriesResult>;
@@ -8,7 +8,7 @@ export type { FilePreviewerInterface, FilePreviewResult } from './filePreviewer'
8
8
  export type { BuildStructureIndexOptions, ListStructureEntriesOptions, ListStructureEntriesResult, ParserInterface } from './parser';
9
9
  export type { ParseStreamOptions, ParseStreamSummary, ParseTextOptions, ParseTextResult, StructureCheckpoint } from './parser';
10
10
  export type { StructureContainerKindId, StructureEntry, StructureIndex, StructureKindId, StructureNode } from './parser';
11
- export type { StructuredParserInterface, StructurePath } from './parser';
11
+ export type { StructuredParserInterface, StructureLocator, StructurePath } from './parser';
12
12
  export { parseDelimitedText, type ParseTextWithDelimiter } from './delimitedText';
13
13
  export declare const TOOL_NAME_PREFIX = "dpuse-tool-";
14
14
  export declare function listParserToolConfigs(toolConfigs: ToolConfig[], dataFormatId: DataFormatId): ToolConfig[];
@@ -15,6 +15,7 @@ export interface ParseTextOptions {
15
15
  valueDelimiterIds: ValueDelimiterId[];
16
16
  }
17
17
  export interface ParseTextResult {
18
+ hasHeaders?: boolean | undefined;
18
19
  issues: SampleIssue[];
19
20
  parsedRecords: ParsedRecord[];
20
21
  recordDelimiterId: RecordDelimiterId;
@@ -22,9 +23,11 @@ export interface ParseTextResult {
22
23
  }
23
24
  export interface ParseStreamOptions {
24
25
  abortSignal: AbortSignal;
26
+ byteLimit?: number | undefined;
25
27
  chunkSize: number | undefined;
26
28
  dataFormatId: DataFormatId;
27
29
  encodingId: string;
30
+ stopSignal?: AbortSignal | undefined;
28
31
  structurePath?: StructurePath | undefined;
29
32
  valueDelimiterId: ValueDelimiterId;
30
33
  }
@@ -32,32 +35,44 @@ export interface ParseStreamSummary extends RetrieveRecordsSummary {
32
35
  byteCount: number;
33
36
  commentLineCount: number;
34
37
  emptyLineCount: number;
38
+ isComplete?: boolean | undefined;
35
39
  lineCount: number;
36
40
  nonUniformRecordCount: number;
37
41
  }
38
42
  export interface BuildStructureIndexOptions {
39
43
  abortSignal: AbortSignal;
40
44
  dataFormatId: DataFormatId;
45
+ encodingId: string;
46
+ isComplete: boolean;
41
47
  maxDepth: number | undefined;
42
48
  onProgress: ((byteCount: number) => void) | undefined;
43
49
  }
44
50
  export interface StructureIndex {
45
51
  byteCount: number;
52
+ isComplete: boolean;
53
+ issues: SampleIssue[];
46
54
  nodes: StructureNode[];
55
+ suggestedStructurePath: StructurePath | undefined;
56
+ tableCandidateCount: number;
47
57
  }
48
58
  export interface StructureNode {
49
59
  checkpoints: StructureCheckpoint[];
50
60
  childCount: number;
51
- endOffset: number;
61
+ isComplete: boolean;
52
62
  isTableCandidate: boolean;
53
63
  keys: string[];
54
64
  kindId: StructureContainerKindId;
65
+ locator: StructureLocator;
55
66
  path: StructurePath;
56
- startOffset: number;
57
67
  }
58
68
  export interface StructureCheckpoint {
59
69
  itemIndex: number;
60
- offset: number;
70
+ locator: StructureLocator;
71
+ }
72
+ export interface StructureLocator {
73
+ endOffset: number | undefined;
74
+ startOffset: number;
75
+ state: string | undefined;
61
76
  }
62
77
  export type StructureContainerKindId = 'list' | 'object';
63
78
  export type StructureKindId = StructureContainerKindId | 'boolean' | 'null' | 'number' | 'string';
@@ -66,21 +81,20 @@ export interface ListStructureEntriesOptions {
66
81
  abortSignal: AbortSignal;
67
82
  dataFormatId: DataFormatId;
68
83
  entryCount: number;
69
- firstItemIndex: number | undefined;
84
+ locator: StructureLocator;
70
85
  maxValueLength: number | undefined;
71
86
  path: StructurePath;
72
- startOffset: number;
73
87
  }
74
88
  export interface ListStructureEntriesResult {
75
89
  entries: StructureEntry[];
76
- isMore: boolean;
90
+ issues: SampleIssue[];
91
+ nextLocator: StructureLocator | undefined;
77
92
  }
78
93
  export interface StructureEntry {
79
94
  childCount: number | undefined;
80
- endOffset: number;
81
95
  isValueCut: boolean;
82
96
  key: number | string;
83
97
  kindId: StructureKindId;
84
- startOffset: number;
98
+ locator: StructureLocator;
85
99
  value: string | undefined;
86
100
  }
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@dpuse/dpuse-shared",
3
- "version": "1.0.214",
3
+ "version": "1.0.215",
4
4
  "private": false,
5
5
  "description": "Common constants, types and utilities used across DPUse projects.",
6
6
  "license": "MIT",