@f-o-t/ofx 2.0.0 → 2.2.0

package/dist/index.js

@@ -343,7 +343,7 @@ var transactionSchema = z.object({
   DTAVAIL: ofxDateSchema.optional(),
   DTPOSTED: ofxDateSchema,
   DTUSER: ofxDateSchema.optional(),
-  FITID: z.string(),
+  FITID: z.string().optional(),
   MEMO: z.string().optional(),
   NAME: z.string().optional(),
   PAYEEID: z.string().optional(),
@@ -360,6 +360,14 @@ var accountTypeSchema = z.enum([
   "CREDITLINE",
   "CD"
 ]);
+var extendedAccountTypeSchema = z.enum([
+  "CHECKING",
+  "SAVINGS",
+  "MONEYMRKT",
+  "CREDITLINE",
+  "CD",
+  "CREDITCARD"
+]);
 var bankAccountSchema = z.object({
   ACCTID: z.string(),
   ACCTKEY: z.string().optional(),
@@ -367,6 +375,13 @@ var bankAccountSchema = z.object({
   BANKID: z.string(),
   BRANCHID: z.string().optional()
 });
+var flexibleBankAccountSchema = z.object({
+  ACCTID: z.string(),
+  ACCTKEY: z.string().optional(),
+  ACCTTYPE: extendedAccountTypeSchema.optional(),
+  BANKID: z.string().optional(),
+  BRANCHID: z.string().optional()
+});
 var creditCardAccountSchema = z.object({
   ACCTID: z.string(),
   ACCTKEY: z.string().optional()
@@ -390,11 +405,14 @@ var bankStatementResponseSchema = z.object({
 });
 var creditCardStatementResponseSchema = z.object({
   AVAILBAL: balanceSchema.optional(),
+  BANKACCTFROM: flexibleBankAccountSchema.optional(),
   BANKTRANLIST: transactionListSchema.optional(),
-  CCACCTFROM: creditCardAccountSchema,
+  CCACCTFROM: creditCardAccountSchema.optional(),
   CURDEF: z.string().default("USD"),
   LEDGERBAL: balanceSchema.optional(),
   MKTGINFO: z.string().optional()
+}).refine((data) => data.CCACCTFROM || data.BANKACCTFROM, {
+  message: "Either CCACCTFROM or BANKACCTFROM is required"
 });
 var signOnResponseSchema = z.object({
   ACCESSKEY: z.string().optional(),
@@ -405,14 +423,14 @@ var signOnResponseSchema = z.object({
   STATUS: statusSchema
 });
 var bankStatementTransactionResponseSchema = z.object({
-  STATUS: statusSchema,
+  STATUS: statusSchema.optional(),
   STMTRS: bankStatementResponseSchema.optional(),
-  TRNUID: z.string()
+  TRNUID: z.string().optional()
 });
 var creditCardStatementTransactionResponseSchema = z.object({
   CCSTMTRS: creditCardStatementResponseSchema.optional(),
-  STATUS: statusSchema,
-  TRNUID: z.string()
+  STATUS: statusSchema.optional(),
+  TRNUID: z.string().optional()
 });
 var singleOrArray = (schema) => z.union([schema, z.array(schema)]).optional();
 var bankMessageSetResponseSchema = z.object({
@@ -446,6 +464,25 @@ var ofxDocumentSchema = z.object({
 });
 
 // src/parser.ts
+var CHARSET_MAP = {
+  "1252": "windows-1252",
+  "WINDOWS-1252": "windows-1252",
+  CP1252: "windows-1252",
+  "8859-1": "iso-8859-1",
+  "ISO-8859-1": "iso-8859-1",
+  LATIN1: "iso-8859-1",
+  "LATIN-1": "iso-8859-1",
+  "UTF-8": "utf-8",
+  UTF8: "utf-8",
+  NONE: "utf-8",
+  "": "utf-8"
+};
+function getEncodingFromCharset(charset) {
+  if (!charset)
+    return "utf-8";
+  const normalized = charset.toUpperCase().trim();
+  return CHARSET_MAP[normalized] ?? "windows-1252";
+}
 var ENTITY_MAP = {
   "&": "&",
   "'": "'",
@@ -513,6 +550,18 @@ function sgmlToObject(sgml) {
   }
   return result;
 }
+function generateFitId(txn, index) {
+  const date = String(txn.DTPOSTED ?? "");
+  const amount = String(txn.TRNAMT ?? "0");
+  const name = String(txn.NAME ?? txn.MEMO ?? "");
+  const input = `${date}:${amount}:${name}:${index}`;
+  let hash = 0;
+  for (let i = 0;i < input.length; i++) {
+    hash = (hash << 5) - hash + input.charCodeAt(i);
+    hash = hash & hash;
+  }
+  return `AUTO${Math.abs(hash).toString(16).toUpperCase().padStart(8, "0")}`;
+}
 function normalizeResponseArray(msgs, responseKey, statementKey) {
   const responses = msgs[responseKey];
   if (!responses)
@@ -522,13 +571,40 @@ function normalizeResponseArray(msgs, responseKey, statementKey) {
     const tranList = stmt?.BANKTRANLIST;
     if (tranList?.STMTTRN !== undefined) {
       tranList.STMTTRN = toArray(tranList.STMTTRN);
+      const transactions = tranList.STMTTRN;
+      transactions.forEach((txn, idx) => {
+        if (!txn.FITID) {
+          txn.FITID = generateFitId(txn, idx);
+        }
+      });
     }
   }
 }
+function normalizeSignOn(data) {
+  const ofx = data.OFX;
+  if (!ofx)
+    return;
+  const signonMsgs = ofx.SIGNONMSGSRSV1;
+  const sonrs = signonMsgs?.SONRS;
+  if (!sonrs)
+    return;
+  const status = sonrs.STATUS;
+  if (!status)
+    return;
+  if (!sonrs.DTSERVER && status.DTSERVER) {
+    sonrs.DTSERVER = status.DTSERVER;
+    delete status.DTSERVER;
+  }
+  if (!sonrs.LANGUAGE && status.LANGUAGE) {
+    sonrs.LANGUAGE = status.LANGUAGE;
+    delete status.LANGUAGE;
+  }
+}
 function normalizeTransactions(data) {
   const ofx = data.OFX;
   if (!ofx)
     return data;
+  normalizeSignOn(data);
   const bankMsgs = ofx.BANKMSGSRSV1;
   if (bankMsgs) {
     normalizeResponseArray(bankMsgs, "STMTTRNRS", "STMTRS");
@@ -614,6 +690,132 @@ function parseOrThrow(content) {
   }
   return result.data;
 }
+function isValidUtf8(buffer) {
+  let i = 0;
+  while (i < buffer.length) {
+    const byte = buffer[i];
+    if (byte === undefined)
+      break;
+    if (byte <= 127) {
+      i++;
+    } else if ((byte & 224) === 192) {
+      const b1 = buffer[i + 1];
+      if (i + 1 >= buffer.length || b1 === undefined || (b1 & 192) !== 128)
+        return false;
+      i += 2;
+    } else if ((byte & 240) === 224) {
+      const b1 = buffer[i + 1];
+      const b2 = buffer[i + 2];
+      if (i + 2 >= buffer.length || b1 === undefined || b2 === undefined || (b1 & 192) !== 128 || (b2 & 192) !== 128)
+        return false;
+      i += 3;
+    } else if ((byte & 248) === 240) {
+      const b1 = buffer[i + 1];
+      const b2 = buffer[i + 2];
+      const b3 = buffer[i + 3];
+      if (i + 3 >= buffer.length || b1 === undefined || b2 === undefined || b3 === undefined || (b1 & 192) !== 128 || (b2 & 192) !== 128 || (b3 & 192) !== 128)
+        return false;
+      i += 4;
+    } else {
+      return false;
+    }
+  }
+  return true;
+}
+function hasUtf8MultiByte(buffer) {
+  for (let i = 0;i < buffer.length; i++) {
+    const byte = buffer[i];
+    if (byte !== undefined && byte > 127) {
+      return true;
+    }
+  }
+  return false;
+}
+function parseHeaderFromBuffer(buffer) {
+  const maxHeaderSize = Math.min(buffer.length, 1000);
+  const headerSection = new TextDecoder("iso-8859-1").decode(buffer.slice(0, maxHeaderSize));
+  const header = {};
+  const singleLineMatch = headerSection.match(/^(OFXHEADER:\d+.*?)(?=<OFX|<\?xml)/is);
+  if (singleLineMatch?.[1]) {
+    const headerPart = singleLineMatch[1];
+    const fieldRegex = /(\w+):([^\s<]+)/g;
+    let fieldMatch = fieldRegex.exec(headerPart);
+    while (fieldMatch !== null) {
+      const key = fieldMatch[1];
+      const value = fieldMatch[2];
+      if (key && value !== undefined) {
+        header[key] = value;
+      }
+      fieldMatch = fieldRegex.exec(headerPart);
+    }
+  } else {
+    const lines = headerSection.split(/\r?\n/);
+    for (const line of lines) {
+      const trimmed = line.trim();
+      if (trimmed.startsWith("<?xml") || trimmed.startsWith("<OFX")) {
+        break;
+      }
+      const match = trimmed.match(/^(\w+):(.*)$/);
+      if (match?.[1] && match[2] !== undefined) {
+        header[match[1]] = match[2];
+      }
+      if (trimmed === "" && Object.keys(header).length > 0) {
+        break;
+      }
+    }
+  }
+  const parsedHeader = ofxHeaderSchema.parse(header);
+  let encoding = getEncodingFromCharset(parsedHeader.CHARSET);
+  if (encoding !== "utf-8" && hasUtf8MultiByte(buffer) && isValidUtf8(buffer)) {
+    encoding = "utf-8";
+  }
+  return { encoding, header: parsedHeader };
+}
+function parseBuffer(buffer) {
+  try {
+    if (!(buffer instanceof Uint8Array)) {
+      return {
+        error: new z2.ZodError([
+          {
+            code: "invalid_type",
+            expected: "object",
+            message: "Expected Uint8Array",
+            path: []
+          }
+        ]),
+        success: false
+      };
+    }
+    if (buffer.length === 0) {
+      return {
+        error: new z2.ZodError([
+          {
+            code: "custom",
+            message: "Buffer cannot be empty",
+            path: []
+          }
+        ]),
+        success: false
+      };
+    }
+    const { encoding } = parseHeaderFromBuffer(buffer);
+    const decoder = new TextDecoder(encoding);
+    const content = decoder.decode(buffer);
+    return parse(content);
+  } catch (err) {
+    if (err instanceof z2.ZodError) {
+      return { error: err, success: false };
+    }
+    throw err;
+  }
+}
+function parseBufferOrThrow(buffer) {
+  const result = parseBuffer(buffer);
+  if (!result.success) {
+    throw result.error;
+  }
+  return result.data;
+}
 // src/stream.ts
 var ENTITY_MAP2 = {
   "&amp;": "&",
@@ -628,7 +830,7 @@ function decodeEntities2(text) {
     return text;
   return text.replace(ENTITY_REGEX2, (match) => ENTITY_MAP2[match] ?? match);
 }
-function parseHeaderFromBuffer(buffer) {
+function parseHeaderFromBuffer2(buffer) {
   const lines = buffer.split(/\r?\n/);
   const header = {};
   let bodyStartIndex = 0;
@@ -672,7 +874,7 @@ function tryParseBalance(obj) {
   const result = balanceSchema.safeParse(obj);
   return result.success ? result.data : null;
 }
-async function* parseStream(input) {
+async function* parseStream(input, options) {
   const state = {
     buffer: "",
     currentObject: {},
@@ -682,7 +884,8 @@ async function* parseStream(input) {
     objectStack: [{}],
     transactionCount: 0
   };
-  const decoder = new TextDecoder;
+  let detectedEncoding = options?.encoding;
+  let decoder = new TextDecoder(detectedEncoding ?? "utf-8");
   const tagRegex = /<(\/?)([\w.]+)>([^<]*)/g;
   let pendingLedgerBalance;
   let pendingAvailableBalance;
@@ -690,10 +893,14 @@ async function* parseStream(input) {
   async function* processChunk(chunk, isLast = false) {
     state.buffer += chunk;
     if (!state.headerParsed) {
-      const headerResult = parseHeaderFromBuffer(state.buffer);
+      const headerResult = parseHeaderFromBuffer2(state.buffer);
       if (headerResult) {
         state.headerParsed = true;
         state.inHeader = false;
+        if (!detectedEncoding && headerResult.header.CHARSET) {
+          detectedEncoding = getEncodingFromCharset(headerResult.header.CHARSET);
+          decoder = new TextDecoder(detectedEncoding);
+        }
         yield { data: headerResult.header, type: "header" };
         state.buffer = state.buffer.slice(headerResult.bodyStart);
       } else {
@@ -794,7 +1001,32 @@ async function* parseStream(input) {
   }
   if (input instanceof ReadableStream) {
     const reader = input.getReader();
+    const initialChunks = [];
+    let headerFound = false;
     try {
+      while (!headerFound) {
+        const { done, value } = await reader.read();
+        if (done)
+          break;
+        initialChunks.push(value);
+        const combined = new Uint8Array(initialChunks.reduce((sum, chunk) => sum + chunk.length, 0));
+        let offset = 0;
+        for (const chunk of initialChunks) {
+          combined.set(chunk, offset);
+          offset += chunk.length;
+        }
+        const headerSection = new TextDecoder("iso-8859-1").decode(combined.slice(0, Math.min(combined.length, 1000)));
+        if (headerSection.includes("<OFX") || headerSection.includes("<?xml")) {
+          const charsetMatch = headerSection.match(/CHARSET:(\S+)/i);
+          if (charsetMatch && !detectedEncoding) {
+            detectedEncoding = getEncodingFromCharset(charsetMatch[1]);
+            decoder = new TextDecoder(detectedEncoding);
+          }
+          headerFound = true;
+          const content = decoder.decode(combined);
+          yield* processChunk(content);
+        }
+      }
       while (true) {
         const { done, value } = await reader.read();
         if (done)
@@ -816,13 +1048,13 @@ async function* parseStream(input) {
   }
   yield { transactionCount: state.transactionCount, type: "complete" };
 }
-async function parseStreamToArray(input) {
+async function parseStreamToArray(input, options) {
   const result = {
     accounts: [],
     balances: [],
     transactions: []
   };
-  for await (const event of parseStream(input)) {
+  for await (const event of parseStream(input, options)) {
     switch (event.type) {
       case "header":
         result.header = event.data;
@@ -840,13 +1072,128 @@ async function parseStreamToArray(input) {
   }
   return result;
 }
+async function* createChunkIterable(content, chunkSize = 65536) {
+  for (let i = 0;i < content.length; i += chunkSize) {
+    yield content.slice(i, i + chunkSize);
+    await new Promise((resolve) => setTimeout(resolve, 0));
+  }
+}
+async function* parseBatchStream(files, options) {
+  let totalTransactions = 0;
+  let errorCount = 0;
+  for (let i = 0;i < files.length; i++) {
+    const file = files[i];
+    if (!file)
+      continue;
+    yield { type: "file_start", fileIndex: i, filename: file.filename };
+    try {
+      let fileTransactionCount = 0;
+      const headerSection = new TextDecoder("iso-8859-1").decode(file.buffer.slice(0, Math.min(file.buffer.length, 1000)));
+      const charsetMatch = headerSection.match(/CHARSET:(\S+)/i);
+      const encoding = charsetMatch ? getEncodingFromCharset(charsetMatch[1]) : options?.encoding ?? "utf-8";
+      const decoder = new TextDecoder(encoding);
+      const content = decoder.decode(file.buffer);
+      const chunkIterable = createChunkIterable(content);
+      for await (const event of parseStream(chunkIterable, options)) {
+        switch (event.type) {
+          case "header":
+            yield { type: "header", fileIndex: i, data: event.data };
+            break;
+          case "transaction":
+            yield { type: "transaction", fileIndex: i, data: event.data };
+            fileTransactionCount++;
+            break;
+          case "account":
+            yield { type: "account", fileIndex: i, data: event.data };
+            break;
+          case "balance":
+            yield { type: "balance", fileIndex: i, data: event.data };
+            break;
+          case "complete":
+            break;
+        }
+      }
+      totalTransactions += fileTransactionCount;
+      yield {
+        type: "file_complete",
+        fileIndex: i,
+        filename: file.filename,
+        transactionCount: fileTransactionCount
+      };
+    } catch (err) {
+      errorCount++;
+      yield {
+        type: "file_error",
+        fileIndex: i,
+        filename: file.filename,
+        error: err instanceof Error ? err.message : String(err)
+      };
+    }
+    await new Promise((resolve) => setTimeout(resolve, 0));
+  }
+  yield {
+    type: "batch_complete",
+    totalFiles: files.length,
+    totalTransactions,
+    errorCount
+  };
+}
+async function parseBatchStreamToArray(files, options) {
+  const results = files.map((file, index) => ({
+    fileIndex: index,
+    filename: file.filename,
+    transactions: [],
+    accounts: [],
+    balances: []
+  }));
+  for await (const event of parseBatchStream(files, options)) {
+    switch (event.type) {
+      case "header": {
+        const result = results[event.fileIndex];
+        if (result)
+          result.header = event.data;
+        break;
+      }
+      case "transaction": {
+        const result = results[event.fileIndex];
+        if (result)
+          result.transactions.push(event.data);
+        break;
+      }
+      case "account": {
+        const result = results[event.fileIndex];
+        if (result)
+          result.accounts.push(event.data);
+        break;
+      }
+      case "balance": {
+        const result = results[event.fileIndex];
+        if (result)
+          result.balances.push(event.data);
+        break;
+      }
+      case "file_error": {
+        const result = results[event.fileIndex];
+        if (result)
+          result.error = event.error;
+        break;
+      }
+    }
+  }
+  return results;
+}
 export {
   parseStreamToArray,
   parseStream,
   parseOrThrow,
+  parseBufferOrThrow,
+  parseBuffer,
+  parseBatchStreamToArray,
+  parseBatchStream,
   parse,
   getTransactions,
   getSignOnInfo,
+  getEncodingFromCharset,
   getBalance,
   getAccountInfo,
   generateHeader,

package/package.json

@@ -51,14 +51,13 @@
       "build": "bunup",
       "check": "biome check --write .",
       "dev": "bunup --watch",
-      "publish": "bunx npm publish",
       "release": "bumpp --commit --push --tag",
       "test": "bun test",
       "test:coverage": "bun test --coverage",
       "test:watch": "bun test --watch",
-      "typecheck": "tsc --noEmit"
+      "typecheck": "tsc "
    },
    "type": "module",
    "types": "./dist/index.d.ts",
-   "version": "2.0.0"
+   "version": "2.2.0"
 }