@dpuse/dpuse-shared 1.0.209 → 1.0.211

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -6,6 +6,7 @@ export interface DataViewConfig extends ComponentInstanceConfig {
6
6
  connectionId: string;
7
7
  connectionNodeConfig: ConnectionNodeConfig | undefined;
8
8
  readingConfig: ReadingConfig | undefined;
9
+ readingChoiceConfig?: ReadingChoiceConfig | undefined;
9
10
  sampleConfig: SampleConfig | undefined;
10
11
  contentAuditConfig: ContentAuditConfig | undefined;
11
12
  columnChoiceConfigs: ColumnChoiceConfigs | undefined;
@@ -22,19 +23,32 @@ export interface ReadingConfig {
22
23
  isDeclared?: boolean | undefined;
23
24
  parserId: string | undefined;
24
25
  recordDelimiterId: RecordDelimiterId | undefined;
25
- replacementCharacterCount?: number | undefined;
26
26
  valueDelimiterId: ValueDelimiterId | undefined;
27
27
  }
28
28
  export type DataFormatId = (typeof DATA_FORMAT_CONFIGS)[number]['id'];
29
- export type DataFormatSourceId = 'choice' | 'fileContent' | 'jsonSyntax' | 'mimeType' | 'extension' | 'default';
30
- export type EncodingSourceId = 'choice' | 'byteOrderMark' | 'xmlDeclaration' | 'xmlDefault' | 'utf8Validation' | 'detection' | 'asciiAssumption' | 'fallback';
29
+ export interface ReadingChoiceConfig {
30
+ dataFormatId: DataFormatId | undefined;
31
+ encodingId: string | undefined;
32
+ hasHeaders: boolean | undefined;
33
+ parserId: string | undefined;
34
+ valueDelimiterId: ValueDelimiterId | undefined;
35
+ }
36
+ export type DataFormatSourceId = 'fileContent' | 'jsonSyntax' | 'mimeType' | 'extension' | 'default';
37
+ export type EncodingSourceId = 'byteOrderMark' | 'xmlDeclaration' | 'xmlDefault' | 'utf8Validation' | 'detection' | 'asciiAssumption' | 'fallback';
31
38
  export type RecordDelimiterId = '\n' | '\r' | '\r\n';
32
39
  export type ValueDelimiterId = (typeof VALUE_DELIMITER_CONFIGS)[number]['id'] | '';
33
40
  export interface SampleConfig {
41
+ dataFormatId?: DataFormatId | undefined;
42
+ decodedEncodingId?: string | undefined;
43
+ hasHeaders?: boolean | undefined;
34
44
  issues: SampleIssue[];
35
45
  parsedRecords: ParsedRecord[];
46
+ parserId?: string | undefined;
47
+ recordDelimiterId?: RecordDelimiterId | undefined;
48
+ replacementCharacterCount?: number | undefined;
36
49
  size: number | undefined;
37
50
  text: string | undefined;
51
+ valueDelimiterId?: ValueDelimiterId | undefined;
38
52
  }
39
53
  export interface SampleIssue {
40
54
  endLineNumber: number | undefined;
@@ -2,7 +2,7 @@
2
2
  * Defines the configuration metadata for a connector. Used for validation
3
3
  * of connector manifests and capability discovery at runtime.
4
4
  */
5
- export declare const connectorActionNameSchema: import('valibot').UnionSchema<readonly [import('valibot').LiteralSchema<"abortOperation", undefined>, import('valibot').LiteralSchema<"auditObjectContent", undefined>, import('valibot').LiteralSchema<"closeStore", undefined>, import('valibot').LiteralSchema<"createObject", undefined>, import('valibot').LiteralSchema<"describeConnection", undefined>, import('valibot').LiteralSchema<"dropObject", undefined>, import('valibot').LiteralSchema<"findObject", undefined>, import('valibot').LiteralSchema<"getInfo", undefined>, import('valibot').LiteralSchema<"getReadableStream", undefined>, import('valibot').LiteralSchema<"getRecord", undefined>, import('valibot').LiteralSchema<"listNodes", undefined>, import('valibot').LiteralSchema<"loadObjectRecords", undefined>, import('valibot').LiteralSchema<"openStore", undefined>, import('valibot').LiteralSchema<"previewObject", undefined>, import('valibot').LiteralSchema<"queryRecords", undefined>, import('valibot').LiteralSchema<"removeRecords", undefined>, import('valibot').LiteralSchema<"retrieveChunks", undefined>, import('valibot').LiteralSchema<"retrieveRecords", undefined>, import('valibot').LiteralSchema<"upsertRecords", undefined>], undefined>;
5
+ export declare const connectorActionNameSchema: import('valibot').UnionSchema<readonly [import('valibot').LiteralSchema<"abortOperation", undefined>, import('valibot').LiteralSchema<"auditObjectContent", undefined>, import('valibot').LiteralSchema<"closeStore", undefined>, import('valibot').LiteralSchema<"createObject", undefined>, import('valibot').LiteralSchema<"describeConnection", undefined>, import('valibot').LiteralSchema<"dropObject", undefined>, import('valibot').LiteralSchema<"findObject", undefined>, import('valibot').LiteralSchema<"getInfo", undefined>, import('valibot').LiteralSchema<"getReadableStream", undefined>, import('valibot').LiteralSchema<"getRecord", undefined>, import('valibot').LiteralSchema<"listNodes", undefined>, import('valibot').LiteralSchema<"loadObjectRecords", undefined>, import('valibot').LiteralSchema<"openStore", undefined>, import('valibot').LiteralSchema<"previewObject", undefined>, import('valibot').LiteralSchema<"queryRecords", undefined>, import('valibot').LiteralSchema<"removeRecords", undefined>, import('valibot').LiteralSchema<"retrieveRecords", undefined>, import('valibot').LiteralSchema<"upsertRecords", undefined>], undefined>;
6
6
  export declare const connectorCategoryConfigSchema: import('valibot').StrictObjectSchema<{
7
7
  readonly label: import('valibot').StrictObjectSchema<{
8
8
  readonly en: import('valibot').StringSchema<undefined>;
@@ -25,7 +25,7 @@ export declare const connectorUsageConfigSchema: import('valibot').StrictObjectS
25
25
  }, undefined>;
26
26
  export declare const connectorConfigSchema: import('valibot').StrictObjectSchema<{
27
27
  readonly typeId: import('valibot').LiteralSchema<"connector", undefined>;
28
- readonly actionNames: import('valibot').ArraySchema<import('valibot').UnionSchema<readonly [import('valibot').LiteralSchema<"abortOperation", undefined>, import('valibot').LiteralSchema<"auditObjectContent", undefined>, import('valibot').LiteralSchema<"closeStore", undefined>, import('valibot').LiteralSchema<"createObject", undefined>, import('valibot').LiteralSchema<"describeConnection", undefined>, import('valibot').LiteralSchema<"dropObject", undefined>, import('valibot').LiteralSchema<"findObject", undefined>, import('valibot').LiteralSchema<"getInfo", undefined>, import('valibot').LiteralSchema<"getReadableStream", undefined>, import('valibot').LiteralSchema<"getRecord", undefined>, import('valibot').LiteralSchema<"listNodes", undefined>, import('valibot').LiteralSchema<"loadObjectRecords", undefined>, import('valibot').LiteralSchema<"openStore", undefined>, import('valibot').LiteralSchema<"previewObject", undefined>, import('valibot').LiteralSchema<"queryRecords", undefined>, import('valibot').LiteralSchema<"removeRecords", undefined>, import('valibot').LiteralSchema<"retrieveChunks", undefined>, import('valibot').LiteralSchema<"retrieveRecords", undefined>, import('valibot').LiteralSchema<"upsertRecords", undefined>], undefined>, undefined>;
28
+ readonly actionNames: import('valibot').ArraySchema<import('valibot').UnionSchema<readonly [import('valibot').LiteralSchema<"abortOperation", undefined>, import('valibot').LiteralSchema<"auditObjectContent", undefined>, import('valibot').LiteralSchema<"closeStore", undefined>, import('valibot').LiteralSchema<"createObject", undefined>, import('valibot').LiteralSchema<"describeConnection", undefined>, import('valibot').LiteralSchema<"dropObject", undefined>, import('valibot').LiteralSchema<"findObject", undefined>, import('valibot').LiteralSchema<"getInfo", undefined>, import('valibot').LiteralSchema<"getReadableStream", undefined>, import('valibot').LiteralSchema<"getRecord", undefined>, import('valibot').LiteralSchema<"listNodes", undefined>, import('valibot').LiteralSchema<"loadObjectRecords", undefined>, import('valibot').LiteralSchema<"openStore", undefined>, import('valibot').LiteralSchema<"previewObject", undefined>, import('valibot').LiteralSchema<"queryRecords", undefined>, import('valibot').LiteralSchema<"removeRecords", undefined>, import('valibot').LiteralSchema<"retrieveRecords", undefined>, import('valibot').LiteralSchema<"upsertRecords", undefined>], undefined>, undefined>;
29
29
  readonly category: import('valibot').NullableSchema<import('valibot').StrictObjectSchema<{
30
30
  readonly label: import('valibot').StrictObjectSchema<{
31
31
  readonly en: import('valibot').StringSchema<undefined>;
@@ -24,9 +24,7 @@ export interface ConnectorInterface {
24
24
  previewObject?(options: PreviewObjectOptions): Promise<PreviewObjectResult>;
25
25
  queryRecords?(options: QueryRecordsOptions): Promise<QueryRecordsResult>;
26
26
  removeRecords?(options: RemoveRecordsOptions): Promise<void>;
27
- retrieveChunks?(options: RetrieveChunksOptions, chunk: (data: Uint8Array) => void, complete: () => void): Promise<void>;
28
- retrieveRecords?(options: RetrieveRecordsOptions, chunk: (typeId: RecordRetrievalTypeId, records: Record<string, unknown>[] | ParsedRecord[]) => void | Promise<void>, // A returned promise holds the read back until it settles.
29
- complete: (result: RetrieveRecordsSummary) => void): Promise<void>;
27
+ retrieveRecords?(options: RetrieveRecordsOptions): AsyncGenerator<RetrieveRecordsChunk, RetrieveRecordsSummary, undefined>;
30
28
  upsertRecords?(options: UpsertRecordsOptions): Promise<void>;
31
29
  }
32
30
  export type ConnectorActionName = InferOutput<typeof connectorActionNameSchema>;
@@ -53,13 +51,13 @@ export interface AuditObjectContentOptions extends EngineConnectorActionOptions
53
51
  byteLimit: number | undefined;
54
52
  chunkSize: number | undefined;
55
53
  dataFormatId: DataFormatId;
56
- encodingId: string;
54
+ encodingId: string | undefined;
57
55
  hasHeaders: boolean | undefined;
58
56
  parserId: string | undefined;
59
57
  path: string;
60
58
  stopSignal: AbortSignal | undefined;
61
59
  supportsTransferableStreams: boolean;
62
- valueDelimiterId: ValueDelimiterId;
60
+ valueDelimiterId: ValueDelimiterId | undefined;
63
61
  }
64
62
  export interface AuditObjectContentResult {
65
63
  contentAuditConfig: ContentAuditConfig;
@@ -67,8 +65,10 @@ export interface AuditObjectContentResult {
67
65
  }
68
66
  export type CloseStoreOptions = EngineConnectorActionOptions;
69
67
  export interface CreateObjectOptions extends EngineConnectorActionOptions {
68
+ columnConfigs: ObjectColumnConfig[];
69
+ indexedColumnIndexes: number[];
70
+ keyColumnIndex: number | undefined;
70
71
  path: string;
71
- structure: string;
72
72
  }
73
73
  export type DescribeConnectionOptions = EngineConnectorActionOptions;
74
74
  interface DescribeConnectionResult {
@@ -91,15 +91,15 @@ export interface GetInfoResult {
91
91
  info: Record<string, unknown>;
92
92
  }
93
93
  export interface GetReadableStreamOptions extends EngineConnectorActionOptions {
94
- id: string;
95
94
  path: string;
96
95
  }
97
96
  export interface GetRecordOptions extends EngineConnectorActionOptions {
98
- id: string;
97
+ id: RecordKey;
99
98
  path: string;
100
99
  }
100
+ export type RecordKey = number | string;
101
101
  export interface GetRecordResult {
102
- record?: string[] | Record<string, unknown>;
102
+ record: StoreRecord;
103
103
  }
104
104
  export interface ListNodesOptions extends EngineConnectorActionOptions {
105
105
  folderPath: string;
@@ -193,44 +193,42 @@ export interface ClassifyPreviewValuesOptions extends EngineConnectorActionOptio
193
193
  }
194
194
  export type PreviewValueStateId = 'blank' | 'empty' | 'invalid' | 'null' | 'valid';
195
195
  export interface RemoveRecordsOptions extends EngineConnectorActionOptions {
196
- keys: string[];
197
- path: string;
198
- }
199
- export interface RetrieveChunksOptions extends EngineConnectorActionOptions {
200
- chunkSize: number | undefined;
201
- encodingId: string;
196
+ keys: RecordKey[];
202
197
  path: string;
203
- valueDelimiterId: ValueDelimiterId;
204
198
  }
205
199
  export interface RetrieveRecordsOptions extends EngineConnectorActionOptions {
206
200
  chunkSize: number | undefined;
207
201
  dataFormatId: DataFormatId;
208
- encodingId: string;
202
+ encodingId: string | undefined;
209
203
  parserId: string | undefined;
210
204
  path: string;
211
205
  range?: RecordRange | undefined;
212
- valueDelimiterId: ValueDelimiterId;
206
+ valueDelimiterId: ValueDelimiterId | undefined;
213
207
  }
214
208
  export interface RecordRange {
215
209
  limit: number;
216
210
  offset: number;
217
211
  }
218
212
  export interface RetrieveRecordsSummary {
219
- byteCount: number;
220
- commentLineCount: number;
221
- emptyLineCount: number;
222
- lineCount: number;
223
- nonUniformRecordCount: number;
224
213
  recordCount: number;
225
214
  totalCount?: number | undefined;
226
215
  }
227
- export type RecordRetrievalTypeId = 'jsonRecordArray' | 'parsingRecordArray';
216
+ export type RecordRetrievalTypeId = 'parsingRecordArray' | 'storeRecordArray';
217
+ export type RetrieveRecordsChunk = ParsedRecordsChunk | {
218
+ records: StoreRecord[];
219
+ typeId: 'storeRecordArray';
220
+ };
221
+ export interface ParsedRecordsChunk {
222
+ records: ParsedRecord[];
223
+ typeId: 'parsingRecordArray';
224
+ }
228
225
  export interface UpsertRecordsOptions extends EngineConnectorActionOptions {
229
- records: Record<string, unknown>[];
226
+ records: StoreRecord[];
230
227
  path: string;
231
228
  }
232
229
  export type ConnectorUsageId = InferOutput<typeof connectorUsageIdSchema>;
233
230
  export { auditObjectContentWithParser, buildPreviewObjectResult, retrieveRecordsWithParser } from './objectParsing';
231
+ export { convertStoreRecordToParsedRecord, convertStoreValueToText } from './storeValues';
234
232
  export { readTableSchema } from './tableSchema';
235
233
  export declare function determineConnectorUsageId(actionNames: ConnectorActionName[]): ConnectorUsageId;
236
234
  export declare const constructConnectorCategoryConfig: (id: string, localeId?: import('../../../locale').LocaleId) => LocalisedConfig<ConnectorCategoryConfig>;
@@ -1,5 +1,5 @@
1
- import { AuditObjectContentOptions, AuditObjectContentResult, ConnectorUtilities, PreviewObjectResult, RecordRetrievalTypeId, RetrieveRecordsOptions, RetrieveRecordsSummary } from '.';
2
- import { DeclaredColumnConfig, DeclaredReadingConfig, ParsedRecord, ValueDelimiterId } from '../../dataView';
1
+ import { AuditObjectContentOptions, AuditObjectContentResult, ConnectorUtilities, ParsedRecordsChunk, PreviewObjectResult, RetrieveRecordsOptions, RetrieveRecordsSummary } from '.';
2
+ import { DeclaredColumnConfig, DeclaredReadingConfig, ReadingChoiceConfig } from '../../dataView';
3
3
  import { FilePreviewResult, ToolConfig } from '../tool';
4
4
  export declare function buildPreviewObjectResult(input: {
5
5
  asAt: number;
@@ -7,18 +7,16 @@ export declare function buildPreviewObjectResult(input: {
7
7
  declaredReadingConfig?: DeclaredReadingConfig | undefined;
8
8
  filePreview: FilePreviewResult;
9
9
  inferDataTypes: ConnectorUtilities['inferDataTypes'];
10
- parserId: string | undefined;
10
+ readingChoiceConfig: ReadingChoiceConfig | undefined;
11
11
  startedAt: number;
12
12
  toolConfigs: ToolConfig[];
13
- valueDelimiterId: ValueDelimiterId | undefined;
14
13
  }): Promise<PreviewObjectResult>;
15
14
  export declare function retrieveRecordsWithParser(input: {
16
- chunk: (typeId: RecordRetrievalTypeId, records: ParsedRecord[]) => void | Promise<void>;
17
15
  options: RetrieveRecordsOptions;
18
16
  signal: AbortSignal;
19
17
  stream: ReadableStream<Uint8Array>;
20
18
  toolConfigs: ToolConfig[];
21
- }): Promise<RetrieveRecordsSummary>;
19
+ }): AsyncGenerator<ParsedRecordsChunk, RetrieveRecordsSummary, undefined>;
22
20
  export declare function auditObjectContentWithParser(input: {
23
21
  constructColumnAuditor: ConnectorUtilities['constructColumnAuditor'];
24
22
  declaredColumnConfigs?: DeclaredColumnConfig[] | undefined;
@@ -0,0 +1,4 @@
1
+ import { ParsedRecord } from '../../dataView';
2
+ import { StoreRecord, StoreValue } from '.';
3
+ export declare function convertStoreRecordToParsedRecord(storeRecord: StoreRecord): ParsedRecord;
4
+ export declare function convertStoreValueToText(storeValue: StoreValue): string | null;
@@ -2,16 +2,13 @@ import { FileTypeResult } from 'file-type';
2
2
  import { EncodingDetectionConfig } from '../../../encoding';
3
3
  import { DataFormatId, DataFormatSourceId, EncodingSourceId } from '../../dataView';
4
4
  export interface FilePreviewerInterface {
5
- previewFile: (url: string, signal: AbortSignal, chunkSize?: number, mimeType?: string, choices?: PreviewChoices) => Promise<FilePreviewResult>;
6
- }
7
- export interface PreviewChoices {
8
- dataFormatId: DataFormatId | undefined;
9
- encodingId: string | undefined;
5
+ previewFile: (url: string, signal: AbortSignal, chunkSize?: number, mimeType?: string, encodingId?: string) => Promise<FilePreviewResult>;
10
6
  }
11
7
  export interface FilePreviewResult {
12
8
  bytes: Uint8Array;
13
9
  dataFormatId: DataFormatId | undefined;
14
10
  dataFormatSourceId: DataFormatSourceId | undefined;
11
+ decodedEncodingId: string | undefined;
15
12
  encodingId: string | undefined;
16
13
  encodingConfidenceLevel: number | undefined;
17
14
  encodingSourceId: EncodingSourceId | undefined;
@@ -4,8 +4,8 @@ import { parserConfigSchema, toolConfigSchema } from './toolConfig.schema';
4
4
  export { toolConfigSchema } from './toolConfig.schema';
5
5
  export type ToolConfig = InferOutput<typeof toolConfigSchema>;
6
6
  export type ParserConfig = InferOutput<typeof parserConfigSchema>;
7
- export type { FilePreviewerInterface, FilePreviewResult, PreviewChoices } from './filePreviewer';
8
- export type { ParserInterface, ParseStreamOptions, ParseTextOptions, ParseTextResult } from './parser';
7
+ export type { FilePreviewerInterface, FilePreviewResult } from './filePreviewer';
8
+ export type { ParserInterface, ParseStreamOptions, ParseStreamSummary, ParseTextOptions, ParseTextResult } from './parser';
9
9
  export { parseDelimitedText, type ParseTextWithDelimiter } from './delimitedText';
10
10
  export declare const TOOL_NAME_PREFIX = "dpuse-tool-";
11
11
  export declare function listParserToolConfigs(toolConfigs: ToolConfig[], dataFormatId: DataFormatId): ToolConfig[];
@@ -1,7 +1,7 @@
1
1
  import { DataFormatId, ParsedRecord, RecordDelimiterId, SampleIssue, ValueDelimiterId } from '../../dataView';
2
- import { RecordRetrievalTypeId, RetrieveRecordsSummary } from '../connector';
2
+ import { ParsedRecordsChunk, RetrieveRecordsSummary } from '../connector';
3
3
  export interface ParserInterface {
4
- parseStream: (stream: ReadableStream<Uint8Array>, options: ParseStreamOptions, chunk: (typeId: RecordRetrievalTypeId, records: ParsedRecord[]) => void | Promise<void>) => Promise<RetrieveRecordsSummary>;
4
+ parseStream: (stream: ReadableStream<Uint8Array>, options: ParseStreamOptions) => AsyncGenerator<ParsedRecordsChunk, ParseStreamSummary, undefined>;
5
5
  parseText: (text: string, options: ParseTextOptions) => Promise<ParseTextResult>;
6
6
  }
7
7
  export interface ParseTextOptions {
@@ -22,3 +22,10 @@ export interface ParseStreamOptions {
22
22
  encodingId: string;
23
23
  valueDelimiterId: ValueDelimiterId;
24
24
  }
25
+ export interface ParseStreamSummary extends RetrieveRecordsSummary {
26
+ byteCount: number;
27
+ commentLineCount: number;
28
+ emptyLineCount: number;
29
+ lineCount: number;
30
+ nonUniformRecordCount: number;
31
+ }
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@dpuse/dpuse-shared",
3
- "version": "1.0.209",
3
+ "version": "1.0.211",
4
4
  "private": false,
5
5
  "description": "Common constants, types and utilities used across DPUse projects.",
6
6
  "license": "MIT",
@@ -73,7 +73,7 @@
73
73
  "fast-check": "^4.10.2",
74
74
  "iconv-lite": "^0.7.3",
75
75
  "jschardet": "^4.1.0",
76
- "prettier": "^3.9.10",
76
+ "prettier": "^3.10.0",
77
77
  "sonda": "^0.14.0",
78
78
  "typescript": "^6.0.3",
79
79
  "vite": "^8.3.8",