@dpuse/dpuse-shared 1.0.183 → 1.0.185

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,5 +1,7 @@
1
+ import { BaseConfig } from '../../index';
1
2
  import { ConnectorConfig } from '../module/connector';
2
3
  import { LocaleLabel } from '../../locale/label';
4
+ import { LocalisedConfig } from '../../locale';
3
5
  import { ComponentInstanceConfig } from '..';
4
6
  import { DataSubtypeId, DataTypeId } from '../dataView';
5
7
  export interface ConnectionConfig extends ComponentInstanceConfig {
@@ -31,28 +33,16 @@ export interface ConnectionDescriptionConfig {
31
33
  columns: ObjectColumnConfig[];
32
34
  }[];
33
35
  }
34
- export interface ConnectionNodeConfig {
35
- id: string;
36
- label: string;
37
- description: string;
38
- icon: string | null;
39
- iconDark: string | null;
36
+ export interface ConnectionNodeConfig extends LocalisedConfig<BaseConfig> {
40
37
  childCount: number | undefined;
41
- childNodes: ConnectionNodeConfig[];
42
38
  extension: string | undefined;
43
39
  folderPath: string;
44
- handle: DPAFileSystemFileHandle | undefined;
45
40
  lastModifiedAt: number | undefined;
46
41
  mimeType: string | undefined;
47
42
  name: string;
48
43
  size: number | undefined;
49
- typeId: NodeTypeId;
50
- }
51
- interface DPAFileSystemFileHandle {
52
- readonly kind: 'file';
53
- getFile(): Promise<File>;
44
+ typeId: 'folder' | 'object';
54
45
  }
55
- type NodeTypeId = 'folder' | 'object';
56
46
  export interface ObjectColumnConfig {
57
47
  blankValueCount: number | undefined;
58
48
  dataTypeId: DataTypeId;
@@ -5,72 +5,66 @@ export interface DataViewConfig extends ComponentInstanceConfig {
5
5
  typeId: 'dataView';
6
6
  connectionId: string;
7
7
  connectionNodeConfig: ConnectionNodeConfig | undefined;
8
- readConfig: ReadConfig | undefined;
9
- previewConfig: PreviewConfig | undefined;
10
- contentAuditConfig: StoredContentAuditConfig | undefined;
11
- columnSetConfig: ColumnSetConfig | undefined;
12
- relationshipsAuditConfig: RelationshipsAuditConfig | undefined;
13
- storeConnectorId?: string | undefined;
8
+ readingConfig: ReadingConfig | undefined;
9
+ sampleConfig: SampleConfig | undefined;
10
+ contentAuditConfig: ContentAuditConfig | undefined;
11
+ columnChoiceConfigs: ColumnChoiceConfigs | undefined;
12
+ storeConnectorId: string | undefined;
14
13
  }
15
- export interface ColumnSetConfig {
16
- asAt: number;
17
- columnConfigs: DataViewColumnConfig[];
18
- }
19
- export interface DataViewColumnConfig extends ObjectColumnConfig {
20
- chosenDataSubtypeId?: DataSubtypeId | undefined;
21
- chosenDataTypeId?: DataTypeId | undefined;
22
- chosenDecimalSeparatorId?: DecimalSeparatorId | undefined;
23
- chosenNullWordsAreEmpty?: boolean | undefined;
24
- }
25
- export interface ReadConfig {
26
- commentMarkId?: string | undefined;
27
- commentMarkOtherCharSeq?: string | undefined;
14
+ export interface ReadingConfig {
28
15
  dataFormatId: DataFormatId;
29
16
  encodingConfidenceLevel: number | undefined;
30
17
  encodingId: string | undefined;
31
18
  fileType: FileTypeResult | undefined;
32
19
  hasHeaders: boolean | undefined;
33
- linesToSkipAtStart?: number | undefined;
34
20
  parserId?: string | undefined;
21
+ recordDelimiterId: RecordDelimiterId | undefined;
22
+ valueDelimiterId: ValueDelimiterId | undefined;
23
+ commentMarkId?: string | undefined;
24
+ commentMarkOtherCharSeq?: string | undefined;
25
+ linesToSkipAtStart?: number | undefined;
35
26
  quoteEscapeChar?: string | undefined;
36
27
  quoteMarkId?: string | undefined;
37
28
  quoteMarkOtherCharSeq?: string | undefined;
38
- recordDelimiterId: RecordDelimiterId | undefined;
39
29
  recordDelimiterOtherCharSeq?: string | undefined;
40
30
  skipEmptyLines?: boolean | undefined;
41
31
  skipLinesWithEmptyValues?: boolean | undefined;
42
32
  skipLinesWithErrors?: boolean | undefined;
43
- valueDelimiterId: ValueDelimiterId | undefined;
44
33
  valueDelimiterOtherCharSeq?: string | undefined;
45
- valueTrimMethodId?: ValueTrimMethodId | undefined;
34
+ valueTrimMethodId?: 'both' | 'left' | 'right' | 'none' | undefined;
46
35
  }
47
- export interface PreviewConfig {
48
- errorMessage?: string | undefined;
49
- issues?: PreviewIssue[];
50
- parsedRecords: ParsingRecord[];
36
+ export type DataFormatId = 'arrow' | 'avro' | 'dtv' | 'json' | 'jsonl' | 'markdown' | 'ods' | 'parquet' | 'sav' | 'sqlite' | 'xlsx' | 'xml' | 'unknown';
37
+ export type RecordDelimiterId = '\n' | '\r' | '\r\n';
38
+ export type ValueDelimiterId = '' | ':' | ',' | '!' | '0x1E' | ';' | ' ' | '\t' | '_' | '0x1F' | '|';
39
+ export type ColumnChoiceConfigs = Partial<Record<string, ColumnChoiceConfig>>;
40
+ export interface ColumnChoiceConfig {
41
+ dataSubtypeId?: DataSubtypeId | undefined;
42
+ dataTypeId?: DataTypeId | undefined;
43
+ decimalSeparatorId?: DecimalSeparatorId | undefined;
44
+ nullWordsAreEmpty?: boolean | undefined;
45
+ }
46
+ export interface SampleConfig {
47
+ issues?: SampleIssue[];
48
+ parsedRecords: ParsedRecord[];
51
49
  size: number | undefined;
52
50
  text: string | undefined;
53
51
  }
54
- export interface PreviewIssue {
52
+ export interface SampleIssue {
55
53
  endLineNumber?: number | undefined;
56
54
  fieldNumber?: number | undefined;
57
55
  id: string;
58
56
  lineNumber: number | undefined;
59
57
  message: string;
60
58
  recordNumber?: number | undefined;
61
- severityId: PreviewIssueSeverityId;
59
+ severityId: SampleIssueSeverityId;
62
60
  value?: string | undefined;
63
61
  }
64
- export type PreviewIssueSeverityId = 'error' | 'info' | 'warning';
65
- export type DataFormatId = 'arrow' | 'avro' | 'dtv' | 'json' | 'jsonl' | 'markdown' | 'ods' | 'parquet' | 'sav' | 'sqlite' | 'xlsx' | 'xml' | 'unknown';
66
- export type ParsingRecord = ParsingResult[];
67
- interface ParsingResult {
62
+ export type SampleIssueSeverityId = 'error' | 'info' | 'warning';
63
+ export type ParsedRecord = ParsedValue[];
64
+ interface ParsedValue {
68
65
  value: string | null;
69
66
  valueWasQuoted: boolean;
70
67
  }
71
- export type RecordDelimiterId = '\n' | '\r' | '\r\n';
72
- export type ValueDelimiterId = '' | ':' | ',' | '!' | '0x1E' | ';' | ' ' | '\t' | '_' | '0x1F' | '|';
73
- type ValueTrimMethodId = 'both' | 'left' | 'right' | 'none';
74
68
  export interface ContentAuditConfig {
75
69
  asAt: number;
76
70
  byteCount: number;
@@ -79,16 +73,11 @@ export interface ContentAuditConfig {
79
73
  emptyLineCount: number | undefined;
80
74
  invalidFieldLengthCount: number | undefined;
81
75
  duration: number;
82
- hasHeaders: boolean;
83
76
  isComplete: boolean;
84
77
  lineCount: number | undefined;
85
78
  recordCount: number;
86
79
  sourceWaitDuration: number | undefined;
87
80
  }
88
- export type StoredContentAuditConfig = Omit<ContentAuditConfig, 'columns' | 'hasHeaders'>;
89
- interface RelationshipsAuditConfig {
90
- placeholder: string;
91
- }
92
81
  export type DataTypeId = 'boolean' | 'numeric' | 'string' | 'temporal' | 'unknown';
93
82
  export type DataSubtypeId = NumericSubtypeId | StringSubtypeId | TemporalSubtypeId;
94
83
  export type NumericSubtypeId = 'bigint' | 'integer' | 'decimal';
@@ -5,7 +5,7 @@ import { LocalisedConfig } from '../../../locale';
5
5
  import { ToolConfig } from '../tool';
6
6
  import { ConnectionConfig, ConnectionDescriptionConfig, ConnectionNodeConfig, ObjectColumnConfig } from '../../connection';
7
7
  import { connectorActionNameSchema, connectorCategoryConfigSchema, connectorConfigSchema, connectorItemTypeConfigSchema, connectorItemTypeIdSchema, connectorUsageConfigSchema, connectorUsageIdSchema } from './connectorConfig.schema';
8
- import { ContentAuditConfig, DataFormatId, InferenceRecord, InferenceSummary, ParsingRecord, PreviewConfig, ReadConfig, ValueDelimiterId } from '../../dataView';
8
+ import { ContentAuditConfig, DataFormatId, InferenceRecord, InferenceSummary, ParsedRecord, ReadingConfig, SampleConfig, ValueDelimiterId } from '../../dataView';
9
9
  export { connectorConfigSchema } from './connectorConfig.schema';
10
10
  export { auditObjectContentWithParser, buildPreviewObjectResult, type ObjectFilePreview, retrieveRecordsWithParser } from './objectParsing';
11
11
  export interface ConnectorInterface extends Component {
@@ -29,7 +29,7 @@ export interface ConnectorInterface extends Component {
29
29
  queryRecords?(options: QueryRecordsOptions): Promise<QueryRecordsResult>;
30
30
  removeRecords?(options: RemoveRecordsOptions): Promise<void>;
31
31
  retrieveChunks?(options: RetrieveChunksOptions, chunk: (data: Uint8Array) => void, complete: () => void): Promise<void>;
32
- retrieveRecords?(options: RetrieveRecordsOptions, chunk: (typeId: RecordRetrievalTypeId, records: Record<string, unknown>[] | ParsingRecord[]) => void | Promise<void>, // A returned promise holds the read back until it settles.
32
+ retrieveRecords?(options: RetrieveRecordsOptions, chunk: (typeId: RecordRetrievalTypeId, records: Record<string, unknown>[] | ParsedRecord[]) => void | Promise<void>, // A returned promise holds the read back until it settles.
33
33
  complete: (result: RetrieveRecordsSummary) => void): Promise<void>;
34
34
  upsertRecords?(options: UpsertRecordsOptions): Promise<void>;
35
35
  }
@@ -38,11 +38,11 @@ export type ConnectorConstructor = new (connectorUtilities: ConnectorUtilities,
38
38
  export interface ConnectorUtilities {
39
39
  constructColumnAuditor: (hasHeaders: boolean | undefined) => ColumnAuditor;
40
40
  hasReadableStreamTransferSupport(): boolean;
41
- inferValues: (parsedRecord: ParsingRecord, columnConfigs: ObjectColumnConfig[], hasLeadingRecord: boolean) => InferenceRecord;
42
- inferDataTypes: (parsedRecords: ParsingRecord[], hasHeaders?: boolean) => InferenceSummary;
41
+ inferValues: (parsedRecord: ParsedRecord, columnConfigs: ObjectColumnConfig[], hasLeadingRecord: boolean) => InferenceRecord;
42
+ inferDataTypes: (parsedRecords: ParsedRecord[], hasHeaders?: boolean) => InferenceSummary;
43
43
  }
44
44
  export interface ColumnAuditor {
45
- auditRecords: (parsedRecords: ParsingRecord[]) => void;
45
+ auditRecords: (parsedRecords: ParsedRecord[]) => void;
46
46
  completeAudit: () => {
47
47
  columnConfigs: ObjectColumnConfig[];
48
48
  hasHeaders: boolean;
@@ -67,6 +67,7 @@ export interface AuditObjectContentOptions extends EngineConnectorActionOptions
67
67
  }
68
68
  export interface AuditObjectContentResult {
69
69
  contentAuditConfig: ContentAuditConfig;
70
+ hasHeaders: boolean;
70
71
  }
71
72
  export type CloseStoreOptions = EngineConnectorActionOptions;
72
73
  export interface CreateObjectOptions extends EngineConnectorActionOptions {
@@ -152,8 +153,8 @@ export interface PreviewObjectOptions extends EngineConnectorActionOptions {
152
153
  }
153
154
  export interface PreviewObjectResult {
154
155
  contentAuditConfig: ContentAuditConfig;
155
- previewConfig: PreviewConfig;
156
- readConfig: ReadConfig;
156
+ sampleConfig: SampleConfig;
157
+ readingConfig: ReadingConfig;
157
158
  }
158
159
  export interface QueryRecordsOptions extends EngineConnectorActionOptions {
159
160
  aggregates: QueryAggregate[];
@@ -185,11 +186,11 @@ export interface QueryRecordsResult {
185
186
  }
186
187
  export interface RecountPreviewColumnsOptions extends EngineConnectorActionOptions {
187
188
  hasHeaders: boolean;
188
- parsedRecords: ParsingRecord[];
189
+ parsedRecords: ParsedRecord[];
189
190
  }
190
191
  export interface ClassifyPreviewValuesOptions extends EngineConnectorActionOptions {
191
192
  columnConfigs: ObjectColumnConfig[];
192
- parsedRecords: ParsingRecord[];
193
+ parsedRecords: ParsedRecord[];
193
194
  }
194
195
  export type PreviewValueStateId = 'blank' | 'invalid' | 'missing' | 'valid';
195
196
  export interface RemoveRecordsOptions extends EngineConnectorActionOptions {
@@ -1,6 +1,6 @@
1
1
  import { FileTypeResult } from 'file-type';
2
- import { AuditObjectContentOptions, ColumnAuditor, ConnectorUtilities, PreviewObjectResult, RecordRetrievalTypeId, RetrieveRecordsOptions, RetrieveRecordsSummary } from '.';
3
- import { ContentAuditConfig, DataFormatId, ParsingRecord, ValueDelimiterId } from '../../dataView';
2
+ import { AuditObjectContentOptions, AuditObjectContentResult, ColumnAuditor, ConnectorUtilities, PreviewObjectResult, RecordRetrievalTypeId, RetrieveRecordsOptions, RetrieveRecordsSummary } from '.';
3
+ import { DataFormatId, ParsedRecord, ValueDelimiterId } from '../../dataView';
4
4
  import { ToolConfig } from '../tool';
5
5
  export interface ObjectFilePreview {
6
6
  bytes: Uint8Array;
@@ -21,7 +21,7 @@ export declare function buildPreviewObjectResult(input: {
21
21
  valueDelimiterId: ValueDelimiterId | undefined;
22
22
  }): Promise<PreviewObjectResult>;
23
23
  export declare function retrieveRecordsWithParser(input: {
24
- chunk: (typeId: RecordRetrievalTypeId, records: ParsingRecord[]) => void | Promise<void>;
24
+ chunk: (typeId: RecordRetrievalTypeId, records: ParsedRecord[]) => void | Promise<void>;
25
25
  options: RetrieveRecordsOptions;
26
26
  signal: AbortSignal;
27
27
  stream: ReadableStream<Uint8Array>;
@@ -34,4 +34,4 @@ export declare function auditObjectContentWithParser(input: {
34
34
  signal: AbortSignal;
35
35
  stream: ReadableStream<Uint8Array>;
36
36
  toolConfigs: ToolConfig[];
37
- }): Promise<ContentAuditConfig>;
37
+ }): Promise<AuditObjectContentResult>;
@@ -1,8 +1,8 @@
1
+ import { ParsedRecord, SampleIssue, ValueDelimiterId } from '../../dataView';
1
2
  import { ParseTextOptions, ParseTextResult } from '.';
2
- import { ParsingRecord, PreviewIssue, ValueDelimiterId } from '../../dataView';
3
3
  export type ParseTextWithDelimiter = (text: string, delimiter: ValueDelimiterId, isComplete: boolean) => Promise<{
4
- issues: PreviewIssue[];
5
- records: ParsingRecord[];
4
+ issues: SampleIssue[];
5
+ records: ParsedRecord[];
6
6
  }>;
7
7
  /**
8
8
  * Reads delimited text for a preview: detects its record delimiter, and which of the candidate value delimiters splits
@@ -1,7 +1,7 @@
1
- import { DataFormatId, ParsingRecord, PreviewIssue, RecordDelimiterId, ValueDelimiterId } from '../../dataView';
1
+ import { DataFormatId, ParsedRecord, RecordDelimiterId, SampleIssue, ValueDelimiterId } from '../../dataView';
2
2
  import { RecordRetrievalTypeId, RetrieveRecordsSummary } from '../connector';
3
3
  export interface ParserInterface {
4
- parseStream: (stream: ReadableStream<Uint8Array>, options: ParseStreamOptions, chunk: (typeId: RecordRetrievalTypeId, records: ParsingRecord[]) => void | Promise<void>) => Promise<RetrieveRecordsSummary>;
4
+ parseStream: (stream: ReadableStream<Uint8Array>, options: ParseStreamOptions, chunk: (typeId: RecordRetrievalTypeId, records: ParsedRecord[]) => void | Promise<void>) => Promise<RetrieveRecordsSummary>;
5
5
  parseText: (text: string, options: ParseTextOptions) => Promise<ParseTextResult>;
6
6
  }
7
7
  export interface ParseTextOptions {
@@ -10,8 +10,8 @@ export interface ParseTextOptions {
10
10
  valueDelimiterIds: ValueDelimiterId[];
11
11
  }
12
12
  export interface ParseTextResult {
13
- issues: PreviewIssue[];
14
- parsedRecords: ParsingRecord[];
13
+ issues: SampleIssue[];
14
+ parsedRecords: ParsedRecord[];
15
15
  recordDelimiterId: RecordDelimiterId;
16
16
  valueDelimiterId: ValueDelimiterId | undefined;
17
17
  }
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@dpuse/dpuse-shared",
3
- "version": "1.0.183",
3
+ "version": "1.0.185",
4
4
  "private": false,
5
5
  "description": "Common constants, types and utilities used across DPUse projects.",
6
6
  "license": "MIT",
@@ -64,19 +64,19 @@
64
64
  "valibot": "^1.5.0"
65
65
  },
66
66
  "devDependencies": {
67
- "@dpuse/dpuse-development": "^1.0.145",
68
- "@dpuse/eslint-config-dpuse": "^1.0.148",
67
+ "@dpuse/dpuse-development": "^1.0.147",
68
+ "@dpuse/eslint-config-dpuse": "^1.0.149",
69
69
  "@types/node": "^26.6.6",
70
70
  "@vitest/coverage-v8": "^5.1.0",
71
71
  "eslint": "^10.14.0",
72
- "fallow": "^3.39.0",
72
+ "fallow": "^3.40.0",
73
73
  "fast-check": "^4.10.2",
74
74
  "iconv-lite": "^0.7.3",
75
75
  "jschardet": "^4.1.0",
76
76
  "prettier": "^3.9.10",
77
77
  "sonda": "^0.14.0",
78
78
  "typescript": "^6.0.3",
79
- "vite": "^8.3.5",
79
+ "vite": "^8.3.6",
80
80
  "vite-plugin-dts": "^5.1.2",
81
81
  "vitest": "^5.1.0"
82
82
  }