@dizzlkheinz/ynab-mcpb 0.18.0 → 0.18.2

package/src/tools/reconciliation/__tests__/executor.integration.test.ts

@@ -78,9 +78,12 @@ describeIntegration('Reconciliation Executor - Bulk Create Integration', () => {
   );
 
   it(
-    'reports duplicates when import IDs already exist',
+    'creates transactions without import_id to allow bank matching',
     { meta: { tier: 'domain', domain: 'reconciliation' } },
     async function () {
+      // Note: import_id is intentionally omitted from reconciliation-created transactions
+      // so they can match with bank-imported transactions. YNAB-side duplicate detection
+      // is no longer used; the reconciliation matcher handles duplicate prevention.
       const analysis = buildIntegrationAnalysis(accountSnapshot, 2, 9);
       const params = buildIntegrationParams(
         accountId,
@@ -88,23 +91,7 @@ describeIntegration('Reconciliation Executor - Bulk Create Integration', () => {
         analysis.summary.target_statement_balance,
       );
 
-      const firstRun = await skipOnRateLimit(
-        () =>
-          executeReconciliation({
-            ynabAPI,
-            analysis,
-            params,
-            budgetId,
-            accountId,
-            initialAccount: accountSnapshot,
-            currencyCode: 'USD',
-          }),
-        this,
-      );
-      if (!firstRun) return;
-      trackCreatedTransactions(firstRun);
-
-      const duplicateAttempt = await skipOnRateLimit(
+      const result = await skipOnRateLimit(
         () =>
           executeReconciliation({
             ynabAPI,
@@ -117,15 +104,14 @@ describeIntegration('Reconciliation Executor - Bulk Create Integration', () => {
           }),
         this,
       );
-      if (!duplicateAttempt) return;
-      if (containsRateLimitFailure(duplicateAttempt)) return;
+      if (!result) return;
+      trackCreatedTransactions(result);
+      if (containsRateLimitFailure(result)) return;
 
-      const duplicateActions = duplicateAttempt.actions_taken.filter(
-        (action) => action.duplicate === true,
-      );
-      expect(duplicateActions.length).toBeGreaterThan(0);
-      expect(duplicateAttempt.bulk_operation_details?.duplicates_detected).toBeGreaterThan(0);
-      expect(duplicateAttempt.summary.transactions_created).toBe(0);
+      // Verify transactions were created successfully
+      expect(result.summary.transactions_created).toBe(2);
+      // Verify no YNAB-side duplicate detection occurred (because no import_id)
+      expect(result.bulk_operation_details?.duplicates_detected).toBe(0);
     },
     60000,
   );

package/src/tools/reconciliation/__tests__/executor.test.ts

@@ -484,9 +484,10 @@ describe('executeReconciliation - bulk create mode', () => {
         account_id: txn.account_id,
         amount: txn.amount,
         date: txn.date,
+        payee_name: txn.payee_name,
+        memo: txn.memo,
         cleared: 'cleared',
         approved: true,
-        import_id: txn.import_id, // Include import_id for correlation
       }));
       return { data: { transactions, duplicate_import_ids: [] } };
     });
@@ -638,11 +639,12 @@ describe('executeReconciliation - bulk create mode', () => {
         account_id: txn.account_id,
         amount: txn.amount,
         date: txn.date,
+        payee_name: txn.payee_name,
+        memo: txn.memo,
         cleared: 'cleared',
         approved: true,
-        import_id: txn.import_id, // Include import_id for correlation
       }));
-      return { data: { transactions } };
+      return { data: { transactions, duplicate_import_ids: [] } };
     });
 
     const result = await executeReconciliation({
@@ -690,33 +692,35 @@ describe('executeReconciliation - bulk create mode', () => {
     ).rejects.toMatchObject({ status: 404 });
   });
 
-  it('flags duplicate transactions returned by YNAB API', async () => {
-    const analysis = buildBulkAnalysis(3, 7);
+  it('creates all transactions without import_id (no YNAB-side duplicate detection)', async () => {
+    // Note: import_id is intentionally omitted from reconciliation-created transactions
+    // so they can match with bank-imported transactions. This means YNAB won't detect
+    // duplicates via import_id - the reconciliation matcher is responsible for that.
+    // Use amount of 100 to avoid early balance halting (tolerance is 10 milliunits)
+    const analysis = buildBulkAnalysis(3, 100);
     const params = buildBulkParams(analysis.summary.target_statement_balance);
     const initialAccount = { ...defaultAccountSnapshot };
     const { api, mocks } = createMockYnabAPI(initialAccount);
 
     mocks.createTransactions.mockImplementation(async (_budgetId, body: any) => {
-      const transactions = (body.transactions ?? []).map((txn: any, index: number) => {
-        if (index === 1) {
-          return undefined;
-        }
-        return {
-          id: `created-${index}`,
-          account_id: txn.account_id,
-          amount: txn.amount,
-          date: txn.date,
-          cleared: 'cleared',
-          approved: true,
-          import_id: txn.import_id, // Include import_id for correlation
-        };
-      });
-      const filtered = transactions.filter(Boolean);
-      const duplicateImportId = body.transactions?.[1]?.import_id;
+      const transactions = (body.transactions ?? []).map((txn: any, index: number) => ({
+        id: `created-${index}`,
+        account_id: txn.account_id,
+        amount: txn.amount,
+        date: txn.date,
+        payee_name: txn.payee_name,
+        memo: txn.memo,
+        cleared: 'cleared',
+        approved: true,
+      }));
+      // Verify no import_id is being sent
+      for (const txn of body.transactions ?? []) {
+        expect(txn.import_id).toBeUndefined();
+      }
       return {
         data: {
-          transactions: filtered,
-          duplicate_import_ids: duplicateImportId ? [duplicateImportId] : [],
+          transactions,
+          duplicate_import_ids: [],
         },
       };
     });
@@ -731,10 +735,9 @@ describe('executeReconciliation - bulk create mode', () => {
       currencyCode: 'USD',
     });
 
-    const duplicateActions = result.actions_taken.filter((action) => action.duplicate === true);
-    expect(duplicateActions).toHaveLength(1);
-    expect(result.bulk_operation_details?.duplicates_detected).toBe(1);
-    expect(result.summary.transactions_created).toBe(2);
+    // Without import_id, YNAB creates all transactions (no duplicate detection)
+    expect(result.bulk_operation_details?.duplicates_detected).toBe(0);
+    expect(result.summary.transactions_created).toBe(3);
   });
 
   it('honors halting logic when balance aligns mid-batch', async () => {
@@ -749,11 +752,12 @@ describe('executeReconciliation - bulk create mode', () => {
         account_id: txn.account_id,
         amount: txn.amount,
         date: txn.date,
+        payee_name: txn.payee_name,
+        memo: txn.memo,
         cleared: 'cleared',
         approved: true,
-        import_id: txn.import_id, // Include import_id for correlation
       }));
-      return { data: { transactions } };
+      return { data: { transactions, duplicate_import_ids: [] } };
     });
 
     const result = await executeReconciliation({
@@ -794,11 +798,12 @@ describe('executeReconciliation - bulk create mode', () => {
         account_id: txn.account_id,
         amount: txn.amount,
         date: txn.date,
+        payee_name: txn.payee_name,
+        memo: txn.memo,
         cleared: 'cleared',
         approved: true,
-        import_id: txn.import_id,
       }));
-      return { data: { transactions } };
+      return { data: { transactions, duplicate_import_ids: [] } };
     });
 
     const result = await executeReconciliation({

package/src/tools/reconciliation/executor.ts

@@ -1,4 +1,3 @@
-import { createHash } from 'crypto';
 import type * as ynab from 'ynab';
 import type { SaveTransaction } from 'ynab/dist/models/SaveTransaction.js';
 import { YNABAPIError, YNABErrorCode } from '../../server/errorHandler.js';
@@ -131,30 +130,6 @@ interface PreparedBulkCreateEntry {
   correlationKey: string;
 }
 
-/**
- * Generates a deterministic import_id for reconciliation-created transactions.
- *
- * Uses a dedicated `YNAB:bulk:` prefix to distinguish reconciliation-created transactions
- * from manual bulk creates. This namespace separation is intentional:
- * - Reconciliation operations are automated and system-generated
- * - Manual bulk creates via create_transactions tool can use custom import_id formats
- * - Both interact with YNAB's global duplicate detection via the same import_id mechanism
- *
- * The hash-based correlation in transactionTools.ts uses `hash:` prefix for correlation
- * (when no import_id provided), which is separate from this import_id generation.
- */
-function generateBulkImportId(
-  accountId: string,
-  date: string,
-  amountMilli: number,
-  payee?: string | null,
-): string {
-  const normalizedPayee = (payee ?? '').trim().toLowerCase();
-  const raw = `${accountId}|${date}|${amountMilli}|${normalizedPayee}`;
-  const digest = createHash('sha256').update(raw).digest('hex').slice(0, 24);
-  return `YNAB:bulk:${digest}`;
-}
-
 function parseISODate(dateStr: string | undefined): Date | undefined {
   if (!dateStr) return undefined;
   const d = new Date(dateStr);
@@ -313,7 +288,7 @@ export async function executeReconciliation(options: ExecutionOptions): Promise<
         memo: truncateMemo(bankTxn.memo),
         cleared: 'cleared',
         approved: true,
-        import_id: generateBulkImportId(accountId, bankTxn.date, amountMilli, bankTxn.payee),
+        // Note: import_id intentionally omitted so transactions can match with bank imports
       };
       const correlationKey = generateCorrelationKey(toCorrelationPayload(saveTransaction));
       return {

package/src/tools/transactionTools.ts

@@ -535,10 +535,7 @@ function finalizeResponse(response: BulkCreateResponse): BulkCreateResponse {
 const ReceiptSplitItemSchema = z
   .object({
     name: z.string().min(1, 'Item name is required'),
-    amount: z
-      .number()
-      .finite('Item amount must be a finite number')
-      .refine((value) => value >= 0, 'Item amount must be zero or greater'),
+    amount: z.number().finite('Item amount must be a finite number'),
     quantity: z
       .number()
       .finite('Quantity must be a finite number')
@@ -571,10 +568,7 @@ export const CreateReceiptSplitTransactionSchema = z
       .finite('Receipt subtotal must be a finite number')
       .refine((value) => value >= 0, 'Receipt subtotal must be zero or greater')
       .optional(),
-    receipt_tax: z
-      .number()
-      .finite('Receipt tax must be a finite number')
-      .refine((value) => value >= 0, 'Receipt tax must be zero or greater'),
+    receipt_tax: z.number().finite('Receipt tax must be a finite number'),
     receipt_total: z
       .number()
       .finite('Receipt total must be a finite number')
@@ -1133,32 +1127,350 @@ function buildItemMemo(item: {
   return item.name;
 }
 
-function distributeTaxProportionally(
-  subtotalMilliunits: number,
-  totalTaxMilliunits: number,
-  categories: ReceiptCategoryCalculation[],
-): void {
-  if (totalTaxMilliunits === 0) {
-    for (const category of categories) category.tax_milliunits = 0;
-    return;
+/**
+ * Constants for smart collapse logic
+ */
+const BIG_TICKET_THRESHOLD_MILLIUNITS = 50000; // $50.00
+const COLLAPSE_THRESHOLD = 5; // Collapse if 5 or more remaining items
+const MAX_ITEMS_PER_MEMO = 5;
+const MAX_MEMO_LENGTH = 150;
+
+/**
+ * Applies smart collapse logic to receipt items according to the specification:
+ * 1. Extract special items (big ticket, returns, discounts)
+ * 2. Apply threshold to remaining items
+ * 3. Collapse by category if needed
+ * 4. Handle tax allocation
+ */
+function applySmartCollapseLogic(
+  categoryCalculations: ReceiptCategoryCalculation[],
+  taxMilliunits: number,
+): SubtransactionInput[] {
+  // Step 1: Extract special items and classify remaining items
+  interface SpecialItem {
+    item: ReceiptCategoryCalculation['items'][0];
+    category_id: string;
+    category_name: string | undefined;
+  }
+
+  interface CategoryItems {
+    category_id: string;
+    category_name: string | undefined;
+    items: ReceiptCategoryCalculation['items'][0][];
+  }
+
+  const specialItems: SpecialItem[] = [];
+  const remainingItemsByCategory: CategoryItems[] = [];
+
+  for (const category of categoryCalculations) {
+    const categorySpecials: ReceiptCategoryCalculation['items'][0][] = [];
+    const categoryRemaining: ReceiptCategoryCalculation['items'][0][] = [];
+
+    for (const item of category.items) {
+      const isNegative = item.amount_milliunits < 0;
+      const unitPrice = item.quantity
+        ? item.amount_milliunits / item.quantity
+        : item.amount_milliunits;
+      const isBigTicket = unitPrice > BIG_TICKET_THRESHOLD_MILLIUNITS;
+
+      if (isNegative || isBigTicket) {
+        categorySpecials.push(item);
+      } else {
+        categoryRemaining.push(item);
+      }
+    }
+
+    // Add specials to the special items list (preserving category order)
+    for (const item of categorySpecials) {
+      specialItems.push({
+        item,
+        category_id: category.category_id,
+        category_name: category.category_name,
+      });
+    }
+
+    // Track remaining items by category
+    if (categoryRemaining.length > 0) {
+      remainingItemsByCategory.push({
+        category_id: category.category_id,
+        category_name: category.category_name,
+        items: categoryRemaining,
+      });
+    }
+  }
+
+  // Step 2: Count total remaining positive items
+  const totalRemainingItems = remainingItemsByCategory.reduce(
+    (sum, cat) => sum + cat.items.length,
+    0,
+  );
+
+  // Step 3: Decide whether to collapse
+  const shouldCollapse = totalRemainingItems >= COLLAPSE_THRESHOLD;
+
+  // Build subtransactions
+  const subtransactions: SubtransactionInput[] = [];
+
+  // Add special items first (returns, discounts, big tickets)
+  for (const special of specialItems) {
+    const memo = buildItemMemo({
+      name: special.item.name,
+      quantity: special.item.quantity,
+      memo: special.item.memo,
+    });
+    const payload: SubtransactionInput = {
+      amount: -special.item.amount_milliunits,
+      category_id: special.category_id,
+    };
+    if (memo) payload.memo = memo;
+    subtransactions.push(payload);
   }
 
-  if (subtotalMilliunits <= 0) {
-    throw new Error('Receipt subtotal must be greater than zero to distribute tax');
+  // Add remaining items (collapsed or itemized)
+  if (shouldCollapse) {
+    // Collapse by category
+    for (const categoryGroup of remainingItemsByCategory) {
+      const collapsedSubtransactions = collapseItemsByCategory(categoryGroup);
+      subtransactions.push(...collapsedSubtransactions);
+    }
+  } else {
+    // Itemize each remaining item individually
+    for (const categoryGroup of remainingItemsByCategory) {
+      for (const item of categoryGroup.items) {
+        const memo = buildItemMemo({
+          name: item.name,
+          quantity: item.quantity,
+          memo: item.memo,
+        });
+        const payload: SubtransactionInput = {
+          amount: -item.amount_milliunits,
+          category_id: categoryGroup.category_id,
+        };
+        if (memo) payload.memo = memo;
+        subtransactions.push(payload);
+      }
+    }
   }
 
-  let allocated = 0;
-  categories.forEach((category, index) => {
-    if (index === categories.length - 1) {
-      category.tax_milliunits = totalTaxMilliunits - allocated;
+  // Step 4: Handle tax allocation
+  const taxSubtransactions = allocateTax(categoryCalculations, taxMilliunits);
+  subtransactions.push(...taxSubtransactions);
+
+  return subtransactions;
+}
+
+/**
+ * Collapses items within a category into groups of up to MAX_ITEMS_PER_MEMO
+ */
+function collapseItemsByCategory(categoryGroup: {
+  category_id: string;
+  category_name: string | undefined;
+  items: ReceiptCategoryCalculation['items'][0][];
+}): SubtransactionInput[] {
+  const subtransactions: SubtransactionInput[] = [];
+  const items = categoryGroup.items;
+
+  let currentBatch: ReceiptCategoryCalculation['items'][0][] = [];
+  let currentBatchTotal = 0;
+
+  for (const item of items) {
+    // Check if we've hit the max items per memo
+    if (currentBatch.length >= MAX_ITEMS_PER_MEMO) {
+      // Flush current batch
+      const memo = buildCollapsedMemo(currentBatch);
+      subtransactions.push({
+        amount: -currentBatchTotal,
+        category_id: categoryGroup.category_id,
+        memo,
+      });
+      currentBatch = [];
+      currentBatchTotal = 0;
+    }
+
+    // Try adding this item to the current batch
+    const testBatch = [...currentBatch, item];
+    const testMemo = buildCollapsedMemo(testBatch);
+
+    if (testMemo.length <= MAX_MEMO_LENGTH) {
+      // Fits - add to batch
+      currentBatch.push(item);
+      currentBatchTotal += item.amount_milliunits;
     } else {
-      const proportionalTax = Math.round(
-        (totalTaxMilliunits * category.subtotal_milliunits) / subtotalMilliunits,
-      );
-      category.tax_milliunits = proportionalTax;
-      allocated += proportionalTax;
+      // Doesn't fit - flush current batch and start new one
+      if (currentBatch.length > 0) {
+        const memo = buildCollapsedMemo(currentBatch);
+        subtransactions.push({
+          amount: -currentBatchTotal,
+          category_id: categoryGroup.category_id,
+          memo,
+        });
+        currentBatch = [item];
+        currentBatchTotal = item.amount_milliunits;
+      } else {
+        // Edge case: single item is too long, use it anyway
+        currentBatch = [item];
+        currentBatchTotal = item.amount_milliunits;
+      }
     }
-  });
+  }
+
+  // Flush remaining batch
+  if (currentBatch.length > 0) {
+    const memo = buildCollapsedMemo(currentBatch);
+    subtransactions.push({
+      amount: -currentBatchTotal,
+      category_id: categoryGroup.category_id,
+      memo,
+    });
+  }
+
+  return subtransactions;
+}
+
+/**
+ * Builds a collapsed memo from a list of items
+ * Format: "Item1 $X.XX, Item2 $Y.YY, Item3 $Z.ZZ"
+ * Truncates with "..." if needed
+ */
+function buildCollapsedMemo(items: ReceiptCategoryCalculation['items'][0][]): string {
+  const parts: string[] = [];
+  let currentLength = 0;
+
+  for (let i = 0; i < items.length; i++) {
+    const item = items[i];
+    if (!item) continue;
+    const amount = milliunitsToAmount(item.amount_milliunits);
+    const itemStr = `${item.name} $${amount.toFixed(2)}`;
+    const separator = i > 0 ? ', ' : '';
+    const testLength = currentLength + separator.length + itemStr.length;
+
+    // Check if adding this item would exceed limit
+    if (parts.length > 0 && testLength + 4 > MAX_MEMO_LENGTH) {
+      // Would exceed - stop here and add "..."
+      break;
+    }
+
+    parts.push(itemStr);
+    currentLength = testLength;
+  }
+
+  let result = parts.join(', ');
+
+  // Add "..." if we didn't include all items
+  if (parts.length < items.length) {
+    result += '...';
+  }
+
+  return result;
+}
+
+/**
+ * Allocates tax across categories
+ * - Positive categories get proportional tax subtransactions
+ * - Negative tax creates a single tax refund subtransaction
+ */
+function allocateTax(
+  categoryCalculations: ReceiptCategoryCalculation[],
+  taxMilliunits: number,
+): SubtransactionInput[] {
+  const subtransactions: SubtransactionInput[] = [];
+
+  // Handle tax = 0
+  if (taxMilliunits === 0) {
+    return subtransactions;
+  }
+
+  // Handle negative tax (refund)
+  if (taxMilliunits < 0) {
+    // Find category with largest return
+    let largestReturnCategory: ReceiptCategoryCalculation | undefined = undefined;
+    let largestReturnAmount = 0;
+
+    for (const category of categoryCalculations) {
+      const categoryReturnAmount = category.items
+        .filter((item) => item.amount_milliunits < 0)
+        .reduce((sum, item) => sum + Math.abs(item.amount_milliunits), 0);
+
+      if (categoryReturnAmount > largestReturnAmount) {
+        largestReturnAmount = categoryReturnAmount;
+        largestReturnCategory = category;
+      }
+    }
+
+    // Default to first category if no returns found
+    if (!largestReturnCategory) {
+      largestReturnCategory = categoryCalculations[0];
+    }
+
+    if (largestReturnCategory) {
+      subtransactions.push({
+        amount: -taxMilliunits,
+        category_id: largestReturnCategory.category_id,
+        memo: 'Tax refund',
+      });
+    }
+
+    return subtransactions;
+  }
+
+  // Positive tax - allocate proportionally to positive categories only
+  const positiveCategorySubtotals = categoryCalculations
+    .map((cat) => ({
+      category: cat,
+      positiveSubtotal: cat.items
+        .filter((item) => item.amount_milliunits > 0)
+        .reduce((sum, item) => sum + item.amount_milliunits, 0),
+    }))
+    .filter((x) => x.positiveSubtotal > 0);
+
+  if (positiveCategorySubtotals.length === 0) {
+    // No positive items, no tax allocation
+    return subtransactions;
+  }
+
+  const totalPositiveSubtotal = positiveCategorySubtotals.reduce(
+    (sum, x) => sum + x.positiveSubtotal,
+    0,
+  );
+
+  // Distribute tax using largest remainder method
+  let allocatedTax = 0;
+  const taxAllocations: {
+    category: ReceiptCategoryCalculation;
+    taxAmount: number;
+  }[] = [];
+
+  for (let i = 0; i < positiveCategorySubtotals.length; i++) {
+    const entry = positiveCategorySubtotals[i];
+    if (!entry) continue;
+
+    const { category, positiveSubtotal } = entry;
+
+    if (i === positiveCategorySubtotals.length - 1) {
+      // Last category gets remainder
+      const taxAmount = taxMilliunits - allocatedTax;
+      if (taxAmount > 0) {
+        taxAllocations.push({ category, taxAmount });
+      }
+    } else {
+      const taxAmount = Math.round((taxMilliunits * positiveSubtotal) / totalPositiveSubtotal);
+      if (taxAmount > 0) {
+        taxAllocations.push({ category, taxAmount });
+        allocatedTax += taxAmount;
+      }
+    }
+  }
+
+  // Create tax subtransactions
+  for (const { category, taxAmount } of taxAllocations) {
+    subtransactions.push({
+      amount: -taxAmount,
+      category_id: category.category_id,
+      memo: `Tax - ${category.category_name ?? 'Uncategorized'}`,
+    });
+  }
+
+  return subtransactions;
 }
 
 export async function handleCreateReceiptSplitTransaction(
@@ -1226,32 +1538,29 @@ export async function handleCreateReceiptSplitTransaction(
     );
   }
 
-  distributeTaxProportionally(subtotalMilliunits, taxMilliunits, categoryCalculations);
-
-  const subtransactions: SubtransactionInput[] = categoryCalculations.flatMap((category) => {
-    const itemSubtransactions: SubtransactionInput[] = category.items.map((item) => {
-      const memo = buildItemMemo({ name: item.name, quantity: item.quantity, memo: item.memo });
-      const payload: SubtransactionInput = {
-        amount: -item.amount_milliunits,
-        category_id: category.category_id,
-      };
-      if (memo) payload.memo = memo;
-      return payload;
-    });
-
-    const taxSubtransaction: SubtransactionInput[] =
-      category.tax_milliunits > 0
-        ? [
-            {
-              amount: -category.tax_milliunits,
-              category_id: category.category_id,
-              memo: `Tax - ${category.category_name ?? 'Uncategorized'}`,
-            },
-          ]
-        : [];
+  // Apply smart collapse logic
+  const subtransactions = applySmartCollapseLogic(categoryCalculations, taxMilliunits);
 
-    return [...itemSubtransactions, ...taxSubtransaction];
-  });
+  // Distribute tax proportionally for receipt_summary (only for positive categories)
+  if (taxMilliunits > 0) {
+    const positiveSubtotal = categoryCalculations.reduce(
+      (sum, cat) => sum + Math.max(0, cat.subtotal_milliunits),
+      0,
+    );
+    if (positiveSubtotal > 0) {
+      let remainingTax = taxMilliunits;
+      const positiveCats = categoryCalculations.filter((cat) => cat.subtotal_milliunits > 0);
+      positiveCats.forEach((cat, index) => {
+        if (index === positiveCats.length - 1) {
+          cat.tax_milliunits = remainingTax;
+        } else {
+          const share = Math.round((cat.subtotal_milliunits / positiveSubtotal) * taxMilliunits);
+          cat.tax_milliunits = share;
+          remainingTax -= share;
+        }
+      });
+    }
+  }
 
   const receiptSummary = {
     subtotal: milliunitsToAmount(subtotalMilliunits),