@dpuse/dpuse-shared 1.0.149 → 1.0.151

package/dist/types/src/component/dataView/index.d.ts

@@ -25,6 +25,7 @@ export interface PreviewConfig {
     issues?: PreviewIssue[];
     linesToSkipAtStart?: number | undefined;
     parsedRecords: ParsingRecord[];
+    parserId?: string | undefined;
     quoteEscapeChar?: string | undefined;
     quoteMarkId?: string | undefined;
     quoteMarkOtherCharSeq?: string | undefined;

package/dist/types/src/component/module/connector/index.d.ts

@@ -7,6 +7,7 @@ import { ConnectionDescriptionConfig, ConnectionNodeConfig, ObjectColumnConfig }
 import { connectorActionNameSchema, connectorCategoryConfigSchema, connectorConfigSchema, connectorItemTypeConfigSchema, connectorItemTypeIdSchema, connectorUsageConfigSchema, connectorUsageIdSchema } from './connectorConfig.schema';
 import { ContentAuditConfig, DataFormatId, InferenceRecord, InferenceSummary, ParsingRecord, PreviewConfig, ValueDelimiterId } from '../../dataView';
 export { connectorConfigSchema } from './connectorConfig.schema';
+export { auditObjectContentWithParser, buildPreviewConfig, type ObjectFilePreview, retrieveRecordsWithParser } from './objectParsing';
 export interface ConnectorInterface extends Component {
     abortController: AbortController | undefined;
     readonly config: ConnectorConfig;
@@ -46,9 +47,10 @@ export type ConnectorItemTypeId = InferOutput<typeof connectorItemTypeIdSchema>;
 type ConnectorUsageConfig = InferOutput<typeof connectorUsageConfigSchema>;
 export interface AuditObjectContentOptions extends EngineConnectorActionOptions {
     chunkSize: number | undefined;
+    dataFormatId: DataFormatId;
     encodingId: string;
     hasHeaders: boolean;
-    parsingToolName: string | undefined;
+    parserId: string | undefined;
     path: string;
     supportsTransferableStreams: boolean;
     valueDelimiterId: ValueDelimiterId;
@@ -109,6 +111,7 @@ export interface PreviewObjectOptions extends EngineConnectorActionOptions {
     encodingId: string | undefined;
     extension: string | undefined;
     mimeType: string | undefined;
+    parserId: string | undefined;
     path: string;
 }
 export interface RemoveRecordsOptions extends EngineConnectorActionOptions {
@@ -123,7 +126,9 @@ export interface RetrieveChunksOptions extends EngineConnectorActionOptions {
 }
 export interface RetrieveRecordsOptions extends EngineConnectorActionOptions {
     chunkSize: number | undefined;
+    dataFormatId: DataFormatId;
     encodingId: string;
+    parserId: string | undefined;
     path: string;
     valueDelimiterId: ValueDelimiterId;
 }

package/dist/types/src/component/module/connector/objectParsing.d.ts

@@ -0,0 +1,36 @@
+import { FileTypeResult } from 'file-type';
+import { AuditObjectContentOptions, ColumnAuditor, ConnectorUtilities, RecordRetrievalTypeId, RetrieveRecordsOptions, RetrieveRecordsSummary } from '.';
+import { ContentAuditConfig, DataFormatId, ParsingRecord, PreviewConfig } from '../../dataView';
+import { ToolConfig } from '../tool';
+export interface ObjectFilePreview {
+    bytes: Uint8Array;
+    dataFormatId: DataFormatId | undefined;
+    encodingConfidenceLevel: number | undefined;
+    encodingId: string | undefined;
+    fileTypeConfig: FileTypeResult | undefined;
+    isComplete: boolean;
+    text: string | undefined;
+}
+export declare function buildPreviewConfig(input: {
+    asAt: number;
+    filePreview: ObjectFilePreview;
+    inferDataTypes: ConnectorUtilities['inferDataTypes'];
+    parserId: string | undefined;
+    startedAt: number;
+    toolConfigs: ToolConfig[];
+}): Promise<PreviewConfig>;
+export declare function retrieveRecordsWithParser(input: {
+    chunk: (typeId: RecordRetrievalTypeId, records: ParsingRecord[]) => void;
+    options: RetrieveRecordsOptions;
+    signal: AbortSignal;
+    stream: ReadableStream<Uint8Array>;
+    toolConfigs: ToolConfig[];
+}): Promise<RetrieveRecordsSummary>;
+export declare function auditObjectContentWithParser(input: {
+    constructColumnAuditor: (hasHeaders: boolean) => ColumnAuditor;
+    onProgress: (recordCount: number) => void;
+    options: AuditObjectContentOptions;
+    signal: AbortSignal;
+    stream: ReadableStream<Uint8Array>;
+    toolConfigs: ToolConfig[];
+}): Promise<ContentAuditConfig>;

package/dist/types/src/component/module/tool/delimitedText.d.ts

@@ -0,0 +1,15 @@
+import { ParseTextOptions, ParseTextResult } from '.';
+import { ParsingRecord, PreviewIssue, ValueDelimiterId } from '../../dataView';
+export type ParseTextWithDelimiter = (text: string, delimiter: ValueDelimiterId, isComplete: boolean) => Promise<{
+    issues: PreviewIssue[];
+    records: ParsingRecord[];
+}>;
+/**
+ * Reads delimited text for a preview: detects its record delimiter, and which of the candidate value delimiters splits
+ * it best. The value delimiter is undefined when the text is not delimited: the best one found does not split most
+ * records into the same number of values, more than one, as a table's rows would be. The records parsed with it are
+ * still returned, so the text can be shown as a table if the user chooses. A delimiter whose parse has issues is only
+ * used when every delimiter's does, because a wrong delimiter can itself cause a quote error. Then the one with the
+ * fewest issues is used, the first on a tie, and its issues are returned, so the user learns why the text cannot be read.
+ */
+export declare function parseDelimitedText(text: string, options: ParseTextOptions, parseTextWithDelimiter: ParseTextWithDelimiter): Promise<ParseTextResult>;

package/dist/types/src/component/module/tool/index.d.ts

@@ -1,7 +1,12 @@
 import { InferOutput } from 'valibot';
+import { DataFormatId } from '../../dataView';
 import { parserConfigSchema, toolConfigSchema } from './toolConfig.schema';
 export { toolConfigSchema } from './toolConfig.schema';
 export type ToolConfig = InferOutput<typeof toolConfigSchema>;
 export type ParserConfig = InferOutput<typeof parserConfigSchema>;
 export type { ParserInterface, ParseStreamOptions, ParseTextOptions, ParseTextResult } from './parser';
+export { parseDelimitedText, type ParseTextWithDelimiter } from './delimitedText';
+export declare const TOOL_NAME_PREFIX = "dpuse-tool-";
+export declare function listParserToolConfigs(toolConfigs: ToolConfig[], dataFormatId: DataFormatId): ToolConfig[];
+export declare function resolveParserToolConfig(toolConfigs: ToolConfig[], dataFormatId: DataFormatId, parserId?: string): ToolConfig;
 export declare function loadTool<T>(toolConfigs: ToolConfig[], toolId: string): Promise<T>;

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@dpuse/dpuse-shared",
-    "version": "1.0.149",
+    "version": "1.0.151",
     "private": false,
     "description": "Common constants, types and utilities used across DPUse projects.",
     "license": "MIT",
@@ -64,7 +64,7 @@
         "valibot": "^1.5.0"
     },
     "devDependencies": {
-        "@dpuse/dpuse-development": "^1.0.133",
+        "@dpuse/dpuse-development": "^1.0.135",
         "@dpuse/eslint-config-dpuse": "^1.0.145",
         "@types/node": "^26.6.5",
         "@vitest/coverage-v8": "^5.0.5",