@0xtorch/csv 0.0.1

package/parseCsvRowsToAccountActions.ts

@@ -0,0 +1,67 @@
+import { greaterThan } from '@0xtorch/big-decimal'
+import type { AccountAction, CryptoCurrency, FiatCurrency } from '@0xtorch/core'
+import { parseNormalActionToAccountActions } from '@0xtorch/core'
+import { parseCsvRowToNormalAction } from './parseCsvRowToNormalAction'
+import type { FormattedRow, Parser } from './types'
+
+type ParseCsvRowsToAccountActionsParameters = {
+  readonly rows: readonly FormattedRow[]
+  readonly service: string
+  readonly parsers: readonly Parser[]
+  readonly symbolAssetMap: { [symbol: string]: string } | undefined
+  readonly accountId: string
+  readonly cryptoCurrencies: readonly CryptoCurrency[]
+  readonly fiatCurrencies: readonly FiatCurrency[]
+}
+
+export const parseCsvRowsToAccountActions = ({
+  rows,
+  service,
+  parsers,
+  symbolAssetMap,
+  accountId,
+  cryptoCurrencies,
+  fiatCurrencies,
+}: ParseCsvRowsToAccountActionsParameters): readonly AccountAction[] => {
+  // marketCap で降順 sort
+  const sortedCryptoCurrencies = cryptoCurrencies.toSorted((a, b) => {
+    if (a.market.marketCapUsd === undefined) {
+      return -1
+    }
+    if (b.market.marketCapUsd === undefined) {
+      return 1
+    }
+    return greaterThan(a.market.marketCapUsd, b.market.marketCapUsd) ? -1 : 1
+  })
+  const actions = rows.flatMap((row) => {
+    const result = parseCsvRowToNormalAction({
+      row,
+      service,
+      parsers,
+      symbolAssetMap,
+      accountId,
+      cryptoCurrencies: sortedCryptoCurrencies,
+      fiatCurrencies,
+    })
+    if (result === undefined) {
+      // TODO action が生成できなかった行を記録したい
+      return []
+    }
+    const { action, source } = result
+    return parseNormalActionToAccountActions({
+      action,
+      accountIds: new Set([accountId]),
+      source,
+    })
+  })
+  const mut_sources: string[] = []
+  const mut_actions: AccountAction[] = []
+  for (const action of actions) {
+    mut_actions.push({
+      ...action,
+      order: mut_sources.filter((source) => source === action.source).length,
+    })
+    mut_sources.push(action.source)
+  }
+  return mut_actions
+}

package/parseCsvText.ts

@@ -0,0 +1,14 @@
+import { parse } from 'papaparse'
+import { z } from 'zod'
+
+const schema = z.record(z.string())
+
+export const parseCsvText = (
+  text: string,
+): readonly { [key: string]: string }[] => {
+  const result = parse(text, {
+    header: true,
+    skipEmptyLines: true,
+  })
+  return result.data.map((row) => schema.parse(row))
+}

package/schemas/formatter.ts

@@ -0,0 +1,13 @@
+import { z } from 'zod'
+
+export const formatterValueTypeSchema = z.union([
+  z.literal('blank'),
+  z.literal('string'),
+  z.literal('big-decimal'),
+  z.literal('big-decimal-positive'),
+  z.literal('big-decimal-negative'),
+  z.literal('iso8601'),
+  z.literal('datetime-jp'),
+])
+
+export const formatterSchema = z.record(z.array(formatterValueTypeSchema))

package/schemas/formatterTypes.ts

@@ -0,0 +1,44 @@
+import { createBigDecimal } from '@0xtorch/big-decimal'
+import { z } from 'zod'
+
+export const blankTextSchema = z.literal('').transform(() => undefined)
+
+export const stringTextSchema = z.string()
+
+export const bigDecimalTextSchema = z
+  .string()
+  .regex(/^-?\d+(\.\d+)?$/)
+  .transform((v) => createBigDecimal(v))
+
+export const bigDecimalPositiveTextSchema = z
+  .string()
+  .regex(/^\d+(\.\d+)?$/)
+  .transform((v) => createBigDecimal(v))
+
+export const bigDecimalNegativeTextSchema = z
+  .string()
+  .regex(/^-\d+(\.\d+)?$/)
+  .transform((v) => createBigDecimal(v))
+
+export const iso8601TextSchema = z
+  .string()
+  .regex(
+    /^\d{4}-?\d\d-?\d\d(?:(T|( ))\d\d(?::?\d\d(?::?\d\d(?:\.\d+)?)?)?( )?(?:Z|[+-]\d\d:?\d\d)?)?$/,
+  )
+  .transform((v) => new Date(v).getTime())
+
+// TODO timezone 追加型 date を導入する
+export const datetimeJpTextSchema = z
+  .string()
+  .regex(/^\d{4}\/\d{2}\/\d{2} \d{2}:\d{2}:\d{2}$/)
+  .transform((v) => new Date(`${v}+09:00`).getTime())
+
+export const formatterValueSchema = z.union([
+  blankTextSchema,
+  stringTextSchema,
+  bigDecimalTextSchema,
+  bigDecimalPositiveTextSchema,
+  bigDecimalNegativeTextSchema,
+  iso8601TextSchema,
+  datetimeJpTextSchema,
+])

package/schemas/index.ts

@@ -0,0 +1 @@
+export { csvFormatSchema } from './schemas'

package/schemas/parser.ts

@@ -0,0 +1,110 @@
+import { normalActionActionSchema } from '@0xtorch/core'
+import { z } from 'zod'
+
+export const conditionComponentSchema = z.object({
+  column: z.string(),
+  pattern: z.union([
+    // string
+    z.string(),
+    // number
+    z.number(),
+    // defined
+    z.object({
+      type: z.literal('defined'),
+    }),
+    // undefined
+    z.object({
+      type: z.literal('undefined'),
+    }),
+    // in
+    z.object({
+      type: z.literal('in'),
+      values: z.array(z.union([z.string(), z.number()])),
+    }),
+    // not-in
+    z.object({
+      type: z.literal('not-in'),
+      values: z.array(z.union([z.string(), z.number()])),
+    }),
+  ]),
+})
+
+export const columnComponentSchema = z.object({
+  type: z.literal('column'),
+  column: z.string(),
+})
+
+export const actionComponentSchema = z.object({
+  column: z.string(),
+  patterns: z.record(normalActionActionSchema),
+})
+
+const serviceComponentSchema = z.object({
+  type: z.literal('service'),
+})
+
+export const textComponentSchema = z.union([
+  // premitive string
+  z.string(),
+  // service
+  serviceComponentSchema,
+  // column
+  columnComponentSchema,
+])
+
+export const nftComponentSchema = z.object({
+  type: z.literal('nft'),
+  id: z.array(textComponentSchema),
+})
+
+export const assetComponentSchema = z.union([
+  // asset id を直接指定
+  z.object({
+    type: z.literal('custom'),
+    assetType: z.union([
+      z.literal('CryptoCurrency'),
+      z.literal('FiatCurrency'),
+      z.literal('Nft'),
+    ]),
+    assetId: z.string(),
+  }),
+  // column と一致する symbol を持つ時価総額最大の asset
+  columnComponentSchema,
+  // nft
+  nftComponentSchema,
+])
+
+export const transferComponentSchema = z.object({
+  direction: z.union([z.literal('in'), z.literal('out')]),
+  // TODO column を指定して値を決定する / 絶対値で値を受け取るようにする
+  amount: z.string(),
+  // TODO column を指定して値を決定する / 原則symbolが一致する時価総額最大の asset を選択する
+  // TODO 別途 symbol と asset の対応表に存在する symbol であればそちらを優先する
+  asset: assetComponentSchema,
+})
+
+// 同一の row でも、 action type によって処理が変わるのでは？
+// 条件を提示して、その条件に合致した場合はこの action を生成する的なルールにした方が良いかも
+
+export const parserSchema = z.object({
+  conditions: z.array(conditionComponentSchema),
+  action: normalActionActionSchema,
+  /** 指定した値群を結合して作成したテキストを代入 */
+  source: z.array(textComponentSchema),
+  /** 指定した column の値を timestamp に代入 */
+  timestamp: z.string(),
+  /** 指定した値群を結合して作成したテキストを代入 */
+  comment: z.array(textComponentSchema).optional(),
+  transfers: z.array(transferComponentSchema),
+  /** 指定した値群を結合して作成したテキストを代入 */
+  crossId: z.array(textComponentSchema).optional(),
+  crossType: z
+    .union([z.literal('start'), z.literal('middle'), z.literal('end')])
+    .optional(),
+  /** 指定した値群を結合して作成したテキストを代入 */
+  loanId: z.array(textComponentSchema).optional(),
+  target: nftComponentSchema.optional(),
+  // TODO crypto asset 対応表を作成する
+})
+
+export const parsersSchema = z.array(parserSchema)

package/schemas/schemas.ts

@@ -0,0 +1,11 @@
+import { z } from 'zod'
+import { formatterSchema } from './formatter'
+import { parsersSchema } from './parser'
+
+export const csvFormatSchema = z.object({
+  id: z.string(),
+  service: z.string(),
+  formatter: formatterSchema,
+  parsers: parsersSchema,
+  symbolAssetMap: z.record(z.string()).optional(),
+})

package/tests/assets.ts

@@ -0,0 +1,28 @@
+import { createBigDecimal } from '@0xtorch/big-decimal'
+import type { CryptoCurrency, FiatCurrency } from '@0xtorch/core'
+import { ethereum, jpy, usd } from '@0xtorch/core'
+
+/** Updated: 2024-05-12 14:00 */
+export const testCryptoCurrencies: CryptoCurrency[] = [
+  {
+    type: 'CryptoCurrency',
+    id: 'bitcoin',
+    name: 'Bitcoin',
+    symbol: 'BTC',
+    icon: undefined,
+    market: {
+      coingeckoId: 'bitcoin',
+      marketCapUsd: createBigDecimal('1,205,095,120,187'),
+    },
+    updatedAt: Date.now(),
+  },
+  {
+    ...ethereum,
+    market: {
+      ...ethereum.market,
+      marketCapUsd: createBigDecimal('352,453,045,218'),
+    },
+  },
+]
+
+export const testFiatCurrencies: FiatCurrency[] = [usd, jpy]

package/tests/coincheck-standard.csv

@@ -0,0 +1,17 @@
+取引日時,取引種別,取引形態,通貨ペア,増加通貨名,増加数量,減少通貨名,減少数量,約定価格/数量,単価,手数料通貨,手数料数量,送付元アドレス,送付先アドレス,登録番号,社名,備考
+2021/07/28 23:36:54,入金,,,JPY,260000.0,,,,,,,,,関東財務局長 第00014号,コインチェック株式会社,
+2021/07/28 23:38:14,購入,自己,ETH/JPY,ETH,0.99656,JPY,260000.0,260000.0,260897.48735651,,,,,関東財務局長 第00014号,コインチェック株式会社,
+2021/07/28 23:51:17,送付,,,,,ETH,0.99156,,,ETH,0.005,,0xF9b5E1202AC637BcfED4EEd026086531c5847D00,関東財務局長 第00014号,コインチェック株式会社,
+2021/07/31 10:16:18,入金,,,JPY,2500000.0,,,,,,,,,関東財務局長 第00014号,コインチェック株式会社,
+2021/07/31 11:08:48,購入,自己,ETH/JPY,ETH,8.9589,JPY,2500000.0,2500000.0,279052.11577314,,,,,関東財務局長 第00014号,コインチェック株式会社,
+2021/07/31 11:20:59,送付,,,,,ETH,8.9539,,,ETH,0.005,,0xF9b5E1202AC637BcfED4EEd026086531c5847D00,関東財務局長 第00014号,コインチェック株式会社,
+2021/10/04 10:25:54,入金,,,JPY,5000000.0,,,,,,,,,関東財務局長 第00014号,コインチェック株式会社,
+2021/10/04 10:46:17,購入,媒介,BTC/JPY,BTC,0.05229,JPY,277980.01938,277980.01938,5316122.0,,,,,関東財務局長 第00014号,コインチェック株式会社,
+2021/10/04 10:46:17,購入,媒介,BTC/JPY,BTC,0.01,JPY,53161.45,53161.45,5316145.0,,,,,関東財務局長 第00014号,コインチェック株式会社,
+2021/10/04 10:46:17,購入,媒介,BTC/JPY,BTC,0.41386996,JPY,2200430.6937312,2200430.6937312,5316720.0,,,,,関東財務局長 第00014号,コインチェック株式会社,
+2021/10/04 10:46:17,購入,媒介,BTC/JPY,BTC,0.03,JPY,159510.0,159510.0,5317000.0,,,,,関東財務局長 第00014号,コインチェック株式会社,
+2021/10/04 10:46:17,購入,媒介,BTC/JPY,BTC,0.05126,JPY,272549.42,272549.42,5317000.0,,,,,関東財務局長 第00014号,コインチェック株式会社,
+2021/10/04 10:46:17,購入,媒介,BTC/JPY,BTC,0.05631,JPY,299400.27,299400.27,5317000.0,,,,,関東財務局長 第00014号,コインチェック株式会社,
+2021/10/04 10:46:17,購入,媒介,BTC/JPY,BTC,0.32664995,JPY,1736797.78415,1736797.78415,5317000.0,,,,,関東財務局長 第00014号,コインチェック株式会社,
+2021/10/04 11:46:50,送付,,,,,BTC,0.001,,,BTC,0.0005,,1Ni8NuJrBmRvpctKcd516GYpUuxNoEis4R,関東財務局長 第00014号,コインチェック株式会社,
+2021/10/04 11:58:31,送付,,,,,BTC,0.93837991,,,BTC,0.0005,,1Ni8NuJrBmRvpctKcd516GYpUuxNoEis4R,関東財務局長 第00014号,コインチェック株式会社,

package/tests/coincheck.ts

@@ -0,0 +1,137 @@
+import type { CsvFormat } from '../types'
+
+export const coincheckStandardFormat = {
+  id: 'coincheck-standard',
+  service: 'coincheck',
+  formatter: {
+    取引日時: ['datetime-jp'],
+    取引種別: ['string'],
+    増加通貨名: ['blank', 'string'],
+    増加数量: ['blank', 'big-decimal-positive'],
+    減少通貨名: ['blank', 'string'],
+    減少数量: ['blank', 'big-decimal-positive'],
+    '約定価格/数量': ['blank', 'big-decimal-positive'],
+    単価: ['blank', 'big-decimal-positive'],
+    手数料通貨: ['blank', 'string'],
+    手数料数量: ['blank', 'big-decimal-positive'],
+    送付元アドレス: ['string'],
+    送付先アドレス: ['string'],
+    備考: ['string'],
+  },
+  parsers: [
+    {
+      conditions: [
+        {
+          column: '取引種別',
+          pattern: {
+            type: 'in',
+            values: ['購入', '売却', '交換'],
+          },
+        },
+        {
+          column: '増加通貨名',
+          pattern: {
+            type: 'defined',
+          },
+        },
+        {
+          column: '増加数量',
+          pattern: {
+            type: 'defined',
+          },
+        },
+        {
+          column: '減少通貨名',
+          pattern: {
+            type: 'defined',
+          },
+        },
+        {
+          column: '減少数量',
+          pattern: {
+            type: 'defined',
+          },
+        },
+      ],
+      action: 'trade',
+      source: [
+        'coincheck_',
+        {
+          type: 'column',
+          column: '取引日時',
+        },
+      ],
+      timestamp: '取引日時',
+      comment: [
+        {
+          type: 'column',
+          column: '備考',
+        },
+      ],
+      transfers: [
+        {
+          direction: 'in',
+          amount: '増加数量',
+          asset: {
+            type: 'column',
+            column: '増加通貨名',
+          },
+        },
+        {
+          direction: 'out',
+          amount: '減少数量',
+          asset: {
+            type: 'column',
+            column: '減少通貨名',
+          },
+        },
+      ],
+    },
+    {
+      conditions: [
+        {
+          column: '取引種別',
+          pattern: '送付',
+        },
+        {
+          column: '手数料通貨',
+          pattern: {
+            type: 'defined',
+          },
+        },
+        {
+          column: '手数料数量',
+          pattern: {
+            type: 'defined',
+          },
+        },
+      ],
+      action: 'fee',
+      source: [
+        'coincheck_',
+        {
+          type: 'column',
+          column: '取引日時',
+        },
+      ],
+      timestamp: '取引日時',
+      comment: [
+        'Transfer to ',
+        {
+          type: 'column',
+          column: '送付先アドレス',
+        },
+      ],
+      transfers: [
+        {
+          direction: 'out',
+          amount: '手数料数量',
+          asset: {
+            type: 'column',
+            column: '手数料通貨',
+          },
+        },
+      ],
+    },
+  ],
+} satisfies CsvFormat

package/types.ts

@@ -0,0 +1,17 @@
+import type { z } from 'zod'
+import type { csvFormatSchema } from './schemas'
+import type { formatterSchema } from './schemas/formatter'
+import type { formatterValueSchema } from './schemas/formatterTypes'
+import type { parserSchema } from './schemas/parser'
+
+export type CsvFormat = z.infer<typeof csvFormatSchema>
+
+export type Formatter = z.infer<typeof formatterSchema>
+
+export type FormatterValue = z.infer<typeof formatterValueSchema>
+
+export type FormattedRow = {
+  [column: string]: FormatterValue
+}
+
+export type Parser = z.infer<typeof parserSchema>