@0xtorch/csv 0.0.34 → 0.0.35

package/_types/hasValidHeaderRow.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"hasValidHeaderRow.d.ts","sourceRoot":"","sources":["../hasValidHeaderRow.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,SAAS,CAAA;AAExC,KAAK,2BAA2B,GAAG;IACjC,IAAI,EAAE,IAAI,GAAG,MAAM,CAAA;IACnB,QAAQ,EAAE,MAAM,CAAA;IAChB,MAAM,EAAE,SAAS,CAAA;CAClB,CAAA;AAED,eAAO,MAAM,iBAAiB,gCAI3B,2BAA2B,qBAsB7B,CAAA"}

package/_types/index.d.ts

@@ -1,8 +1,11 @@
+export { detectFileEncoding } from './detectFileEncoding';
 export { formatCsvRows } from './formatCsvRows';
+export { hasValidHeaderRow } from './hasValidHeaderRow';
 export { parseCsvFileToActions } from './parseCsvFileToActions';
 export { parseCsvFileToText } from './parseCsvFileToText';
 export { parseCsvText } from './parseCsvText';
 export { parseRowsToActions } from './parseRowsToActions';
 export { csvFormatSchema } from './schemas';
+export { streamCsvToActions } from './streamCsvToActions';
 export type { CsvFormat } from './types';
 //# sourceMappingURL=index.d.ts.map

package/_types/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAA;AAC/C,OAAO,EAAE,qBAAqB,EAAE,MAAM,yBAAyB,CAAA;AAC/D,OAAO,EAAE,kBAAkB,EAAE,MAAM,sBAAsB,CAAA;AACzD,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAA;AAC7C,OAAO,EAAE,kBAAkB,EAAE,MAAM,sBAAsB,CAAA;AACzD,OAAO,EAAE,eAAe,EAAE,MAAM,WAAW,CAAA;AAC3C,YAAY,EAAE,SAAS,EAAE,MAAM,SAAS,CAAA"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,kBAAkB,EAAE,MAAM,sBAAsB,CAAA;AACzD,OAAO,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAA;AAC/C,OAAO,EAAE,iBAAiB,EAAE,MAAM,qBAAqB,CAAA;AACvD,OAAO,EAAE,qBAAqB,EAAE,MAAM,yBAAyB,CAAA;AAC/D,OAAO,EAAE,kBAAkB,EAAE,MAAM,sBAAsB,CAAA;AACzD,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAA;AAC7C,OAAO,EAAE,kBAAkB,EAAE,MAAM,sBAAsB,CAAA;AACzD,OAAO,EAAE,eAAe,EAAE,MAAM,WAAW,CAAA;AAC3C,OAAO,EAAE,kBAAkB,EAAE,MAAM,sBAAsB,CAAA;AACzD,YAAY,EAAE,SAAS,EAAE,MAAM,SAAS,CAAA"}

package/_types/parseRowsToActions/index.d.ts

@@ -1,6 +1,6 @@
 import type { Action, CryptoCurrency, FiatCurrency } from '@0xtorch/core';
 import type { z } from 'zod';
-import type { parsersWithKeySchema } from '../schemas/parser';
+import type { parserSchema, parsersWithKeySchema } from '../schemas/parser';
 import type { FormattedRow } from '../types';
 type ParseRowsToActionsParameters = {
     rows: readonly FormattedRow[];
@@ -18,5 +18,17 @@ type ParseRowsToActionsReturnTypes = {
     ignoreRowCount: number;
 };
 export declare const parseRowsToActions: ({ rows, parser, symbolAssetMap, service, cryptoes, fiats, errorLogger, }: ParseRowsToActionsParameters) => ParseRowsToActionsReturnTypes;
+type ParseTargetRowsToActionsParameters = {
+    rows: readonly FormattedRow[];
+    parsers: readonly z.infer<typeof parserSchema>[];
+    symbolAssetMap?: {
+        readonly [symbol: string]: string;
+    };
+    service: string;
+    cryptoes: readonly CryptoCurrency[];
+    fiats: readonly FiatCurrency[];
+    errorLogger?: (error: unknown) => void;
+};
+export declare const parseTargetRowsToActions: (parameters: ParseTargetRowsToActionsParameters) => readonly Action[];
 export {};
 //# sourceMappingURL=index.d.ts.map

package/_types/parseRowsToActions/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../parseRowsToActions/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAE,cAAc,EAAE,YAAY,EAAE,MAAM,eAAe,CAAA;AACzE,OAAO,KAAK,EAAE,CAAC,EAAE,MAAM,KAAK,CAAA;AAC5B,OAAO,KAAK,EAAgB,oBAAoB,EAAE,MAAM,mBAAmB,CAAA;AAC3E,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,UAAU,CAAA;AAI5C,KAAK,4BAA4B,GAAG;IAClC,IAAI,EAAE,SAAS,YAAY,EAAE,CAAA;IAC7B,MAAM,EAAE,CAAC,CAAC,KAAK,CAAC,OAAO,oBAAoB,CAAC,CAAA;IAC5C,cAAc,CAAC,EAAE;QAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,GAAG,MAAM,CAAA;KAAE,CAAA;IACtD,OAAO,EAAE,MAAM,CAAA;IACf,QAAQ,EAAE,SAAS,cAAc,EAAE,CAAA;IACnC,KAAK,EAAE,SAAS,YAAY,EAAE,CAAA;IAC9B,WAAW,CAAC,EAAE,CAAC,KAAK,EAAE,OAAO,KAAK,IAAI,CAAA;CACvC,CAAA;AAED,KAAK,6BAA6B,GAAG;IACnC,OAAO,EAAE,SAAS,MAAM,EAAE,CAAA;IAC1B,cAAc,EAAE,MAAM,CAAA;CACvB,CAAA;AAED,eAAO,MAAM,kBAAkB,6EAQ5B,4BAA4B,KAAG,6BAuEjC,CAAA"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../parseRowsToActions/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAE,cAAc,EAAE,YAAY,EAAE,MAAM,eAAe,CAAA;AACzE,OAAO,KAAK,EAAE,CAAC,EAAE,MAAM,KAAK,CAAA;AAC5B,OAAO,KAAK,EAAE,YAAY,EAAE,oBAAoB,EAAE,MAAM,mBAAmB,CAAA;AAC3E,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,UAAU,CAAA;AAI5C,KAAK,4BAA4B,GAAG;IAClC,IAAI,EAAE,SAAS,YAAY,EAAE,CAAA;IAC7B,MAAM,EAAE,CAAC,CAAC,KAAK,CAAC,OAAO,oBAAoB,CAAC,CAAA;IAC5C,cAAc,CAAC,EAAE;QAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,GAAG,MAAM,CAAA;KAAE,CAAA;IACtD,OAAO,EAAE,MAAM,CAAA;IACf,QAAQ,EAAE,SAAS,cAAc,EAAE,CAAA;IACnC,KAAK,EAAE,SAAS,YAAY,EAAE,CAAA;IAC9B,WAAW,CAAC,EAAE,CAAC,KAAK,EAAE,OAAO,KAAK,IAAI,CAAA;CACvC,CAAA;AAED,KAAK,6BAA6B,GAAG;IACnC,OAAO,EAAE,SAAS,MAAM,EAAE,CAAA;IAC1B,cAAc,EAAE,MAAM,CAAA;CACvB,CAAA;AAED,eAAO,MAAM,kBAAkB,6EAQ5B,4BAA4B,KAAG,6BAuEjC,CAAA;AAED,KAAK,kCAAkC,GAAG;IACxC,IAAI,EAAE,SAAS,YAAY,EAAE,CAAA;IAC7B,OAAO,EAAE,SAAS,CAAC,CAAC,KAAK,CAAC,OAAO,YAAY,CAAC,EAAE,CAAA;IAChD,cAAc,CAAC,EAAE;QAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,GAAG,MAAM,CAAA;KAAE,CAAA;IACtD,OAAO,EAAE,MAAM,CAAA;IACf,QAAQ,EAAE,SAAS,cAAc,EAAE,CAAA;IACnC,KAAK,EAAE,SAAS,YAAY,EAAE,CAAA;IAC9B,WAAW,CAAC,EAAE,CAAC,KAAK,EAAE,OAAO,KAAK,IAAI,CAAA;CACvC,CAAA;AAED,eAAO,MAAM,wBAAwB,eACvB,kCAAkC,KAC7C,SAAS,MAAM,EAoBjB,CAAA"}

package/_types/streamCsvToActions.d.ts

@@ -0,0 +1,20 @@
+import type { Action, CryptoCurrency, FiatCurrency } from '@0xtorch/core';
+import type { CsvFormat } from './types';
+type StreamCsvToActionsParameters = {
+    file: File | string;
+    encoding: string;
+    format: CsvFormat;
+    cryptoes: readonly CryptoCurrency[];
+    fiats: readonly FiatCurrency[];
+    save: (actions: readonly Action[]) => Promise<void>;
+    chunkSize?: number;
+};
+type StreamCsvToActionsReturnTypes = {
+    success: boolean;
+    error?: unknown;
+    rowCount: number;
+    ignoreRowCount: number;
+};
+export declare const streamCsvToActions: ({ file, encoding, format, cryptoes, fiats, save, chunkSize, }: StreamCsvToActionsParameters) => Promise<StreamCsvToActionsReturnTypes>;
+export {};
+//# sourceMappingURL=streamCsvToActions.d.ts.map

package/_types/streamCsvToActions.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"streamCsvToActions.d.ts","sourceRoot":"","sources":["../streamCsvToActions.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAE,cAAc,EAAE,YAAY,EAAE,MAAM,eAAe,CAAA;AAMzE,OAAO,KAAK,EAAE,SAAS,EAAgB,MAAM,SAAS,CAAA;AAEtD,KAAK,4BAA4B,GAAG;IAClC,IAAI,EAAE,IAAI,GAAG,MAAM,CAAA;IACnB,QAAQ,EAAE,MAAM,CAAA;IAChB,MAAM,EAAE,SAAS,CAAA;IACjB,QAAQ,EAAE,SAAS,cAAc,EAAE,CAAA;IACnC,KAAK,EAAE,SAAS,YAAY,EAAE,CAAA;IAC9B,IAAI,EAAE,CAAC,OAAO,EAAE,SAAS,MAAM,EAAE,KAAK,OAAO,CAAC,IAAI,CAAC,CAAA;IACnD,SAAS,CAAC,EAAE,MAAM,CAAA;CACnB,CAAA;AAID,KAAK,6BAA6B,GAAG;IACnC,OAAO,EAAE,OAAO,CAAA;IAChB,KAAK,CAAC,EAAE,OAAO,CAAA;IACf,QAAQ,EAAE,MAAM,CAAA;IAChB,cAAc,EAAE,MAAM,CAAA;CACvB,CAAA;AAED,eAAO,MAAM,kBAAkB,kEAQ5B,4BAA4B,KAAG,OAAO,CAAC,6BAA6B,CAsItE,CAAA"}

package/detectFileEncoding.ts

@@ -0,0 +1,15 @@
+import type { ReadableStream } from 'node:stream/web'
+import { detect } from 'encoding-japanese'
+
+export const detectFileEncoding = async (
+  fileStream: ReadableStream,
+): Promise<string> => {
+  const reader = fileStream.getReader()
+  const result = await reader.read()
+  await reader.cancel()
+  const detected = detect(result.value)
+  if (typeof detected === 'string') {
+    return detected
+  }
+  return detected.valueOf().toString()
+}

package/formatCsvRows.ts

@@ -25,7 +25,7 @@ export const formatCsvRows = ({
 }: FormatCsvRowsParameters): readonly FormattedRow[] =>
   rows.map((row) => formatCsvRow(row, formatter))
 
-const formatCsvRow = (row: RawRow, formatter: Formatter): FormattedRow =>
+export const formatCsvRow = (row: RawRow, formatter: Formatter): FormattedRow =>
   Object.fromEntries(
     Object.entries(formatter).map(
       ([column, schemaTypes]): [string, FormatterValue] => {

package/hasValidHeaderRow.ts

@@ -0,0 +1,36 @@
+import { parse } from 'papaparse'
+import type { CsvFormat } from './types'
+
+type HasValidHeaderRowParameters = {
+  file: File | string
+  encoding: string
+  format: CsvFormat
+}
+
+export const hasValidHeaderRow = async ({
+  file,
+  encoding,
+  format,
+}: HasValidHeaderRowParameters) => {
+  let isValid = false
+  parse(file, {
+    header: false,
+    skipEmptyLines: true,
+    encoding,
+    step: (result, parser) => {
+      if (!Array.isArray(result.data)) {
+        parser.abort()
+        return
+      }
+      for (const key of Object.keys(format.formatter)) {
+        if (!result.data.includes(key)) {
+          parser.abort()
+          return
+        }
+      }
+      isValid = true
+      parser.abort()
+    },
+  })
+  return isValid
+}

package/index.ts

@@ -1,7 +1,10 @@
+export { detectFileEncoding } from './detectFileEncoding'
 export { formatCsvRows } from './formatCsvRows'
+export { hasValidHeaderRow } from './hasValidHeaderRow'
 export { parseCsvFileToActions } from './parseCsvFileToActions'
 export { parseCsvFileToText } from './parseCsvFileToText'
 export { parseCsvText } from './parseCsvText'
 export { parseRowsToActions } from './parseRowsToActions'
 export { csvFormatSchema } from './schemas'
+export { streamCsvToActions } from './streamCsvToActions'
 export type { CsvFormat } from './types'

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@0xtorch/csv",
-  "version": "0.0.34",
+  "version": "0.0.35",
   "description": "Cryptorch CSV extension",
   "keywords": [
     "cryptorch",

package/parseRowsToActions/index.ts

@@ -111,7 +111,7 @@ type ParseTargetRowsToActionsParameters = {
   errorLogger?: (error: unknown) => void
 }
 
-const parseTargetRowsToActions = (
+export const parseTargetRowsToActions = (
   parameters: ParseTargetRowsToActionsParameters,
 ): readonly Action[] => {
   const { parsers, errorLogger, ...otherParameters } = parameters

package/streamCsvToActions.ts

@@ -0,0 +1,181 @@
+import type { Action, CryptoCurrency, FiatCurrency } from '@0xtorch/core'
+import { parse } from 'papaparse'
+import { z } from 'zod'
+import { formatCsvRow } from './formatCsvRows'
+import { parseTargetRowsToActions } from './parseRowsToActions'
+import { createKeyValue } from './parseRowsToActions/key'
+import type { CsvFormat, FormattedRow } from './types'
+
+type StreamCsvToActionsParameters = {
+  file: File | string
+  encoding: string
+  format: CsvFormat
+  cryptoes: readonly CryptoCurrency[]
+  fiats: readonly FiatCurrency[]
+  save: (actions: readonly Action[]) => Promise<void>
+  chunkSize?: number
+}
+
+const recordSchema = z.record(z.string())
+
+type StreamCsvToActionsReturnTypes = {
+  success: boolean
+  error?: unknown
+  rowCount: number
+  ignoreRowCount: number
+}
+
+export const streamCsvToActions = async ({
+  file,
+  encoding,
+  format,
+  cryptoes,
+  fiats,
+  save,
+  chunkSize = 100,
+}: StreamCsvToActionsParameters): Promise<StreamCsvToActionsReturnTypes> => {
+  // stream で処理
+  let success = true
+  let error: unknown
+  let ignoreRowCount = 0
+  let rowCount = 0
+  let currentKeyValue: string | undefined
+  let currentRows: FormattedRow[] = []
+  let actions: Action[] = []
+  parse(file, {
+    header: true,
+    skipEmptyLines: true,
+    encoding,
+    step: async (result, parsers) => {
+      rowCount += 1
+
+      // format
+      const row = recordSchema.parse(result.data)
+      const formattedRow = formatRow(row, format.formatter)
+      if (formattedRow === undefined) {
+        console.log('ignore:', row)
+        ignoreRowCount += 1
+        return
+      }
+
+      // parse
+      if (format.parser.key === undefined) {
+        currentRows = []
+        const rowActions = parseTargetRowsToActions({
+          rows: [formattedRow],
+          parsers: format.parser.parsers,
+          symbolAssetMap: format.symbolAssetMap,
+          service: format.service,
+          cryptoes,
+          fiats,
+        })
+        if (rowActions.length === 0) {
+          console.log('ignore:', row)
+          ignoreRowCount += 1
+        }
+        actions.push(...rowActions)
+      } else {
+        const keyValue = createKeyValue({
+          row: formattedRow,
+          key: format.parser.key,
+        })
+        if (
+          currentKeyValue !== undefined &&
+          currentKeyValue !== keyValue &&
+          currentRows.length > 0
+        ) {
+          const keyActions = parseTargetRowsToActions({
+            rows: currentRows,
+            parsers: format.parser.parsers,
+            symbolAssetMap: format.symbolAssetMap,
+            service: format.service,
+            cryptoes,
+            fiats,
+          })
+          if (keyActions.length === 0) {
+            ignoreRowCount += currentRows.length
+          }
+          actions.push(...keyActions)
+          currentRows = []
+        }
+        currentKeyValue = keyValue
+        currentRows.push(formattedRow)
+      }
+
+      // actions が chunkSize に達したら最後の action source と同一 source を除いて actions を保存
+      if (actions.length >= chunkSize + 1) {
+        const lastActionSource = actions[actions.length - 1].source
+        const savingActions: Action[] = []
+        const remainingActions: Action[] = []
+        for (const action of actions) {
+          if (action.source === lastActionSource) {
+            remainingActions.push(action)
+          } else {
+            savingActions.push(action)
+          }
+        }
+        actions = remainingActions
+        try {
+          await save(savingActions)
+        } catch (e) {
+          success = false
+          error = e
+          parsers.abort()
+        }
+      }
+    },
+  })
+
+  if (!success) {
+    return {
+      success: false,
+      error,
+      rowCount,
+      ignoreRowCount,
+    }
+  }
+
+  // currentRows が残っていたら actions に追加
+  if (currentRows.length > 0) {
+    const keyActions = parseTargetRowsToActions({
+      rows: currentRows,
+      parsers: format.parser.parsers,
+      symbolAssetMap: format.symbolAssetMap,
+      service: format.service,
+      cryptoes,
+      fiats,
+    })
+    if (keyActions.length === 0) {
+      ignoreRowCount += currentRows.length
+    }
+    actions.push(...keyActions)
+  }
+
+  // actions が残っていたら actions 保存処理
+  if (actions.length > 0) {
+    try {
+      await save(actions)
+    } catch (e) {
+      success = false
+      error = e
+    }
+  }
+
+  return {
+    success,
+    error,
+    rowCount,
+    ignoreRowCount,
+  }
+}
+
+const formatRow = (
+  row: Record<string, string>,
+  formatter: CsvFormat['formatter'],
+): FormattedRow | undefined => {
+  try {
+    return formatCsvRow(row, formatter)
+  } catch (error) {
+    return undefined
+  }
+}