@prostgles/prostgles 2.3.6-dev.1786548258

package/e2e/tests/testAskLLM/scenarios/geoQuestion.scenario.ts

@@ -0,0 +1,110 @@
+import { makeToolUse, type Scenario } from "testAskLLM/utils";
+
+const sql = `WITH o30 AS (
+  SELECT o.id,
+         o.deliverer_id,
+         a.id AS address_id,
+         a.postal_code,
+         a.country
+  FROM orders o
+  JOIN addresses a ON a.id = o.customer_address_id::bigint
+  WHERE o.created_at >= now() - interval '30 days'
+),
+normalized AS (
+  SELECT id,
+         deliverer_id,
+         address_id,
+         CASE
+           WHEN postal_code ~* '^[A-Z]{1,2}[0-9][0-9A-Z]?\s?[0-9][A-Z]{2}$' THEN upper(replace(postal_code, ' ', ''))
+           WHEN country ~* '^[A-Z]{1,2}[0-9][0-9A-Z]?\s?[0-9][A-Z]{2}$' THEN upper(replace(country, ' ', ''))
+           ELSE NULL
+         END AS full_pc
+  FROM o30
+),
+zone AS (
+  SELECT id,
+         deliverer_id,
+         address_id,
+         full_pc,
+         CASE WHEN full_pc IS NOT NULL AND length(full_pc) > 3 THEN left(full_pc, length(full_pc)-3) ELSE NULL END AS district
+  FROM normalized
+),
+agg AS (
+  SELECT district,
+         MIN(address_id) AS sample_address_id,
+         COUNT(*) AS orders_30d,
+         COUNT(*) FILTER (WHERE deliverer_id IS NULL) AS unassigned_orders_30d,
+         COUNT(*) FILTER (WHERE deliverer_id IS NOT NULL) AS assigned_orders_30d,
+         COUNT(DISTINCT deliverer_id) FILTER (WHERE deliverer_id IS NOT NULL) AS active_riders_30d
+  FROM zone
+  WHERE district IS NOT NULL
+  GROUP BY district
+)
+SELECT district,
+       sample_address_id,
+       orders_30d,
+       unassigned_orders_30d,
+       ROUND(unassigned_orders_30d::numeric / NULLIF(orders_30d,0), 4) AS unassigned_rate,
+       active_riders_30d,
+       ROUND(assigned_orders_30d::numeric / NULLIF(active_riders_30d,0), 2) AS assigned_orders_per_rider,
+       CASE
+         WHEN orders_30d >= 50 AND (unassigned_orders_30d::numeric / NULLIF(orders_30d,0)) >= 0.10 THEN 'low'
+         WHEN orders_30d >= 50 AND ROUND(assigned_orders_30d::numeric / NULLIF(active_riders_30d,0), 2) > 8 THEN 'low'
+         WHEN orders_30d >= 50 THEN 'ok'
+         ELSE 'small_sample'
+       END AS coverage_signal
+FROM agg
+WHERE orders_30d >= 30
+ORDER BY unassigned_rate DESC, assigned_orders_per_rider DESC, orders_30d DESC
+LIMIT 25;`;
+
+const interpretation = `I checked this using the last **30 days** of orders and a normalized UK postcode extraction (since \`addresses.postal_code\` is often polluted and many postcodes are in \`addresses.country\`).
+
+### Result
+With a minimum of **30 orders/zone**, there are currently **no delivery zones showing low rider coverage** under basic coverage signals:
+
+- **Unassigned order rate:** \`0%\` in returned zones  
+- **Assigned orders per active rider (zone-level):** roughly \`~1.00–1.03\`
+
+So based on current data, I don’t see a clearly under-covered zone right now.
+
+### Important caveat
+Your address data is inconsistent (many rows have \`postal_code='England'\`), so postcode normalization was necessary. This can hide true zone-level imbalance.
+
+---
+
+If you want, I can run a stricter operational definition of “low coverage,” e.g.:
+1) high order volume + low unique riders,  
+2) high delivery delay (if timestamps/status history allow),  
+3) rider-to-order imbalance by hour/daypart.`;
+
+export const geoQuestionScenario: Scenario = {
+  firstMessage: "Which delivery zones have low rider coverage?",
+  steps: [
+    {
+      content:
+        "I'll quickly check available tools and refresh the live schema so I can identify “delivery zones” accurately and compute rider coverage correctly.",
+      tool: [
+        makeToolUse("prostgles-ui", "get_tool_list", {}),
+        makeToolUse("db", "get_existing_tables_schema", {
+          tableNames: [],
+          tableNameRegex: "",
+        }),
+      ],
+    },
+    {
+      content:
+        "I'm going to use a better coverage metric: high unassigned-order rate + high orders per active rider, grouped by postcode district (outward code), which is more meaningful than full postcode.",
+      tool: [
+        makeToolUse("db", "execute_readonly_sql", {
+          sql,
+        }),
+      ],
+    },
+    {
+      content: interpretation,
+      tool: [],
+      // result_content: interpretation,
+    },
+  ],
+};

package/e2e/tests/testAskLLM/scenarios/hygieneRatingsApi.scenario.ts

@@ -0,0 +1,62 @@
+import { makeToolUse, type Scenario } from "testAskLLM/utils";
+import { demoRestaurantName } from "utils/constants";
+
+export const hygieneRatingsApiScenario = {
+  firstMessage:
+    "I need to get the hygiene ratings for each restaurant from the gov website",
+  steps: [
+    {
+      tool: [
+        makeToolUse("prostgles-ui", "get_tool_list", {}),
+        makeToolUse("db", "get_existing_tables_schema", {
+          tableNames: [],
+          tableNameRegex: "",
+        }),
+        // makeToolUse("db", "count", {
+        //   tableName: "restaurants",
+        // }),
+        // makeToolUse("db", "execute_readonly_sql", {
+        //   sql: `SELECT r.id, r.name, r.address, a.postal_code, a.city FROM restaurants r LEFT JOIN addresses a ON a.id = r.address_id ORDER BY r.id LIMIT 10;`,
+        // }),
+      ],
+    },
+    {
+      tool: [
+        makeToolUse("db", "count", {
+          tableName: "restaurants",
+        }),
+        makeToolUse("db", "execute_readonly_sql", {
+          sql: `SELECT r.id, r.name, r.address, a.postal_code, a.city FROM restaurants r LEFT JOIN addresses a ON a.id = r.address_id ORDER BY r.id LIMIT 10;`,
+        }),
+      ],
+    },
+    {
+      tool: [
+        makeToolUse("prostgles-ui", "ask_user_questions", {
+          questions: [
+            {
+              type: "choice",
+              question: "How should I deliver the hygiene ratings?",
+              suggestedAnswers: [
+                "Create a new table and save ratings for all restaurants (recommended)",
+                "Just return a preview report (no DB writes)",
+              ],
+            },
+          ],
+        }),
+      ],
+    },
+    {
+      tool: [
+        makeToolUse("prostgles-ui", "create_agentic_workflow", {
+          workflow_function_definition:
+            'import { defineAgenticWorkflow } from "./defineAgenticWorkflow";\n\ntype FsaEstablishment = {\n  BusinessName?: string;\n  RatingValue?: string;\n  RatingDate?: string;\n  FHRSID?: number;\n  LocalAuthorityName?: string;\n  PostCode?: string;\n  AddressLine1?: string;\n  AddressLine2?: string;\n  AddressLine3?: string;\n  AddressLine4?: string;\n  geocode?: { latitude?: string; longitude?: string };\n  scores?: {\n    Hygiene?: number;\n    Structural?: number;\n    ConfidenceInManagement?: number;\n  };\n};\n\ntype FsaResponse = { establishments?: FsaEstablishment[] };\n\nconst normalizeText = (value: string): string =>\n  value.toLowerCase().replace(/[^a-z0-9]+/g, " ").replace(/\\s+/g, " ").trim();\n\nconst sleep = async (ms: number): Promise<void> => {\n  await new Promise((resolve) => {\n    setTimeout(resolve, ms);\n  });\n};\n\nconst computeMatchScore = (\n  restaurantName: string,\n  city: string,\n  postcode: string,\n  candidate: FsaEstablishment,\n): number => {\n  const rn = normalizeText(restaurantName);\n  const cn = normalizeText(candidate.BusinessName ?? "");\n  let score = 0;\n\n  if (rn.length > 0 && cn.length > 0) {\n    if (rn === cn) score += 70;\n    else if (rn.includes(cn) || cn.includes(rn)) score += 40;\n\n    const rt = new Set(rn.split(" ").filter(Boolean));\n    const ct = new Set(cn.split(" ").filter(Boolean));\n    let overlap = 0;\n    for (const token of rt) {\n      if (ct.has(token)) overlap += 1;\n    }\n    score += Math.min(20, overlap * 4);\n  }\n\n  const rp = normalizeText(postcode);\n  const cp = normalizeText(candidate.PostCode ?? "");\n  if (rp.length > 0 && cp.length > 0 && rp === cp) score += 30;\n\n  const cityNorm = normalizeText(city);\n  if (cityNorm.length > 0) {\n    const addr = normalizeText(\n      [candidate.AddressLine1, candidate.AddressLine2, candidate.AddressLine3, candidate.AddressLine4]\n        .filter((v): v is string => Boolean(v))\n        .join(" "),\n    );\n    if (addr.includes(cityNorm)) score += 12;\n  }\n\n  return score;\n};\n\nconst fetchFsaCandidates = async (\n  name: string,\n  city: string,\n  postcode: string,\n): Promise<FsaEstablishment[]> => {\n  const endpoint = "https://api.ratings.food.gov.uk/Establishments";\n  const queries: URL[] = [];\n\n  const q1 = new URL(endpoint);\n  q1.searchParams.set("name", name);\n  q1.searchParams.set("pageSize", "25");\n  if (postcode.trim().length > 0) q1.searchParams.set("address", postcode);\n  else if (city.trim().length > 0) q1.searchParams.set("address", city);\n  queries.push(q1);\n\n  const q2 = new URL(endpoint);\n  q2.searchParams.set("name", name);\n  q2.searchParams.set("pageSize", "25");\n  queries.push(q2);\n\n  const all: FsaEstablishment[] = [];\n\n  for (const query of queries) {\n    let response: Response | null = null;\n    let attempts = 0;\n\n    while (response === null && attempts < 4) {\n      const retry = attempts;\n      attempts += 1;\n      try {\n        const res = await fetch(query.toString(), {\n          headers: { "x-api-version": "2", Accept: "application/json" },\n        });\n        if (res.status === 429 || res.status >= 500) {\n          await sleep(300 * 2 ** retry);\n          continue;\n        }\n        response = res;\n      } catch {\n        await sleep(300 * 2 ** retry);\n      }\n    }\n\n    if (response === null || !response.ok) continue;\n    const data = (await response.json()) as FsaResponse;\n    for (const est of data.establishments ?? []) all.push(est);\n    if (all.length > 0) break;\n  }\n\n  return all;\n};\n\nvoid defineAgenticWorkflow(\n  {\n    name: "Fetch and store restaurant hygiene ratings (UK FSA)",\n    containerConfiguration: {\n      timeout: 14_400_000,\n      internetAccess: "bridge",\n      memory: "1g",\n      cpus: "1",\n    },\n    databaseAccessDefinitions: {\n      mode: "custom",\n      ddlStatements: `\nCREATE TABLE IF NOT EXISTS hygiene_ratings (\n  id BIGSERIAL PRIMARY KEY,\n  restaurant_id int4 NOT NULL REFERENCES restaurants(id) ON DELETE CASCADE,\n  business_name text,\n  rating_value text,\n  rating_value_desc TEXT GENERATED ALWAYS AS (trim(lower(rating_value)) || \' - \' || CASE trim(lower(rating_value)) WHEN \'5\' THEN \'very good\'   WHEN \'4\' THEN \'good\'  WHEN \'3\' THEN \'generally satisfactory\'  WHEN \'2\' THEN \'some improvement necessary\'  WHEN \'1\' THEN \'major improvement necessary\'  WHEN \'0\' THEN \'urgent improvement necessary\'  ELSE trim(lower(rating_value)) END) STORED,\n  hygiene_score int4,\n  structural_score int4,\n  confidence_in_management_score int4,\n  rating_date date,\n  fhrsid int8,\n  local_authority_name text,\n  match_confidence numeric(5,2),\n  match_method text,\n  source_url text NOT NULL DEFAULT \'https://api.ratings.food.gov.uk/Establishments\',\n  fetched_at timestamptz NOT NULL DEFAULT now(),\n  status text NOT NULL DEFAULT \'matched\',\n  error_message text,\n  latitude numeric,\n  longitude numeric,\n  geog geography GENERATED ALWAYS AS (\n    CASE\n      WHEN latitude IS NOT NULL AND longitude IS NOT NULL\n      THEN st_point(longitude::double precision, latitude::double precision, 4326)::geography\n      ELSE NULL::geography\n    END\n  ) STORED,\n  api_payload jsonb,\n  CONSTRAINT hygiene_ratings_restaurant_id_key UNIQUE (restaurant_id)\n);\n      `,\n      tablePermissions: {\n        restaurants: { select: { fields: { id: 1, name: 1, address_id: 1 } } },\n        addresses: { select: true },\n        hygiene_ratings: { select: true, insert: true, update: true },\n      },\n    },\n  },\n  async ({ tableHandlers, setProgress }) => {\n    const apiSourceUrl = "https://api.ratings.food.gov.uk/Establishments";\n    const pageSize = 250;\n\n    await setProgress(1, "Loading processed restaurants...");\n\n    const existing = await tableHandlers.hygiene_ratings.find(\n      {},\n      { select: { restaurant_id: 1 }, limit: 500000 },\n    );\n    const processed = new Set<number>();\n    for (const row of existing) processed.add(row.restaurant_id);\n\n    const total = await tableHandlers.restaurants.count({});\n    await setProgress(3, `Restaurants in DB: ${total}. Already processed: ${processed.size}.`);\n\n    let offset = 0;\n    let doneThisRun = 0;\n\n    while (offset < 5 && offset < total) {\n      const restaurants = await tableHandlers.restaurants.find(\n        { $or: ["' +
+            demoRestaurantName +
+            '", "McDonald","KFC","Burger King","Subway","Domino","Pizza Hut","Starbucks","Costa","Pret","Greggs","Nando"].map(v => ({ name: { $ilike: `%${v}%`} })) },\n        {\n          select: { id: 1, name: 1, address_id: 1 },\n          orderBy: { key: "id", asc: true },\n          limit: pageSize,\n          offset,\n        },\n      );\n      if (restaurants.length === 0) break;\n      offset += restaurants.length;\n\n      const addressIds = Array.from(new Set(restaurants.map((r) => r.address_id)));\n      const addresses = await tableHandlers.addresses.find(\n        { id: { $in: addressIds } },\n        { select: { id: 1, city: 1, postal_code: 1 }, limit: addressIds.length + 20 },\n      );\n      const byAddressId = new Map<string, { city: string; postal_code: string }>();\n      for (const addr of addresses) {\n        byAddressId.set(addr.id, { city: addr.city, postal_code: addr.postal_code });\n      }\n\n      for (const restaurant of restaurants) {\n        if (processed.has(restaurant.id)) continue;\n\n        const address = byAddressId.get(restaurant.address_id);\n        const city = address?.city ?? "";\n        const postcode = address?.postal_code ?? "";\n\n        let status = "not_found";\n        let businessName: string | null = null;\n        let ratingValue: string | null = null;\n        let hygieneScore: number | null = null;\n        let structuralScore: number | null = null;\n        let cimScore: number | null = null;\n        let ratingDate: string | null = null;\n        let fhrsid: number | null = null;\n        let localAuthority: string | null = null;\n        let matchConfidence: number | null = null;\n        let latitude: number | null = null;\n        let longitude: number | null = null;\n        let payload: FsaEstablishment | null = null;\n        let errorMessage: string | null = null;\n\n        try {\n          const candidates = await fetchFsaCandidates(restaurant.name, city, postcode);\n\n          let best: FsaEstablishment | null = null;\n          let bestScore = -1;\n          for (const candidate of candidates) {\n            const score = computeMatchScore(restaurant.name, city, postcode, candidate);\n            if (score > bestScore) {\n              best = candidate;\n              bestScore = score;\n            }\n          }\n\n          if (best !== null && bestScore >= 35) {\n            const lat = best.geocode?.latitude ? Number(best.geocode.latitude) : null;\n            const lon = best.geocode?.longitude ? Number(best.geocode.longitude) : null;\n\n            status = "matched";\n            businessName = best.BusinessName ?? null;\n            ratingValue = best.RatingValue ?? null;\n            hygieneScore = best.scores?.Hygiene ?? null;\n            structuralScore = best.scores?.Structural ?? null;\n            cimScore = best.scores?.ConfidenceInManagement ?? null;\n            ratingDate = best.RatingDate ? best.RatingDate.slice(0, 10) : null;\n            fhrsid = best.FHRSID ?? null;\n            localAuthority = best.LocalAuthorityName ?? null;\n            matchConfidence = bestScore;\n            latitude = Number.isFinite(lat) ? lat : null;\n            longitude = Number.isFinite(lon) ? lon : null;\n            payload = best;\n          } else {\n            status = "not_found";\n            matchConfidence = bestScore < 0 ? 0 : bestScore;\n          }\n        } catch (error) {\n          status = "error";\n          errorMessage = error instanceof Error ? error.message.slice(0, 500) : "Unknown error";\n        }\n\n        await tableHandlers.hygiene_ratings.insert(\n          {\n            restaurant_id: restaurant.id,\n            business_name: businessName,\n            rating_value: ratingValue,\n            hygiene_score: hygieneScore,\n            structural_score: structuralScore,\n            confidence_in_management_score: cimScore,\n            rating_date: ratingDate,\n            fhrsid,\n            local_authority_name: localAuthority,\n            match_confidence: matchConfidence,\n            match_method: "name+address",\n            source_url: apiSourceUrl,\n            fetched_at: new Date().toISOString(),\n            status,\n            error_message: errorMessage,\n            latitude,\n            longitude,\n            api_payload: payload,\n          },\n          { onConflict: "DoUpdate", returning: { restaurant_id: 1 } },\n        );\n\n        processed.add(restaurant.id);\n        doneThisRun += 1;\n\n        if (doneThisRun % 25 === 0) {\n          const complete = processed.size;\n          const progress = Math.min(99, Math.round((complete / total) * 100));\n          await setProgress(\n            progress,\n            `Processed ${doneThisRun} in this run. Total complete ${complete}/${total}.`,\n          );\n        }\n      }\n    }\n\n    await setProgress(100, `Done. Processed ${doneThisRun} restaurants in this run.`);\n  },\n);\n',
+          workflow_function_definition_summary:
+            "Creates a new `hygiene_ratings` table, then fetches UK Food Standards Agency ratings for each restaurant, matches results by name + postcode/city, and upserts one rating row per restaurant. It is rerun-safe (skips already processed rows), stores confidence/status/raw payload, and includes a generated PostGIS geography column from API coordinates for map views.",
+        }),
+      ],
+    },
+  ],
+} as const satisfies Scenario;

package/e2e/tests/testAskLLM/scenarios/receiptImport/receiptImport.scenario.ts

@@ -0,0 +1,53 @@
+import { makeToolUse, type Scenario } from "testAskLLM/utils";
+
+export const receiptImport: Scenario = {
+  firstMessage: "I need to import some pdf receipts",
+  steps: [
+    {
+      content: "",
+      tool: [
+        makeToolUse("prostgles-ui", "ask_user_questions", {
+          questions: [
+            {
+              type: "choice",
+              question: "How should the import run by default?",
+              suggestedAnswers: [
+                "Preview extracted receipts first, then insert approved rows (recommended)",
+                "Insert automatically without preview",
+              ],
+            },
+            {
+              type: "choice",
+              question: "How should likely duplicates be handled?",
+              suggestedAnswers: [
+                "Skip likely duplicates (vendor + date + total + receipt_number)",
+                "Insert all rows even if duplicates",
+                "Update existing matching rows",
+              ],
+            },
+            {
+              type: "choice",
+              question:
+                "If a required field is missing from a PDF, what should happen?",
+              suggestedAnswers: [
+                "Skip that file and report it (recommended)",
+                "Try best-effort inference and insert with notes",
+                "Stop the whole run with an error",
+              ],
+            },
+          ],
+        }),
+      ],
+    },
+    {
+      tool: [
+        makeToolUse("prostgles-ui", "create_agentic_workflow", {
+          workflow_function_definition:
+            'import { defineAgenticWorkflow } from "./defineAgenticWorkflow";\nimport { promises as fs } from "node:fs";\nimport path from "node:path";\n\nvoid defineAgenticWorkflow(\n  {\n    name: "Import PDF Receipts into receipts",\n    containerConfiguration: {\n      timeout: 600000,\n      readOnly: false,\n      internetAccess: "none",\n      memory: "1g",\n      cpus: "1",\n    },\n    databaseAccessDefinitions: {\n      mode: "custom",\n      tablePermissions: {\n        receipts: {\n          select: true,\n          insert: true,\n          update: true,\n        },\n      },\n    },\n    orchestrationTools: {\n      documents: {\n        get_document_text: 1,\n      },\n    },\n    userInput: {\n      sourcePaths: {\n        title: "PDF files or folders",\n        type: "file-or-folder-paths",\n        accessMode: "read",\n      },\n      importMode: {\n        title: "Import mode",\n        type: "enum",\n        values: ["preview_then_insert", "insert_automatically"],\n        defaultValue: "insert_automatically",\n      },\n      duplicateHandling: {\n        title: "Duplicate handling",\n        type: "enum",\n        values: ["skip", "insert_all", "update_existing"],\n        defaultValue: "skip",\n      },\n      missingFieldHandling: {\n        title: "Missing required fields",\n        type: "enum",\n        values: ["skip_file", "infer_best_effort", "fail_run"],\n        defaultValue: "skip_file",\n      },\n      defaultCurrency: {\n        title: "Default currency if not found",\n        type: "enum",\n        values: ["USD"],\n        defaultValue: "USD",\n      },\n    },\n    agentDefinitions: {\n      receiptExtractor: {\n        prompt:\n          "You extract receipt data from OCR text. Return only what can be supported by text. Amounts must be numeric. Dates must be YYYY-MM-DD where possible. If uncertain, return null for that field.",\n        outputSchema: {\n          vendor_name: { type: "string", optional: true },\n          purchase_date: { type: "string", optional: true },\n          currency_code: { type: "string", optional: true },\n          subtotal: { type: "number", optional: true },\n          tax_amount: { type: "number", optional: true },\n          total_amount: { type: "number", optional: true },\n          receipt_number: { type: "string", optional: true },\n          payment_method: { type: "string", optional: true },\n          notes: { type: "string", optional: true },\n          confidence: { type: "number", optional: true },\n        },\n      },\n    },\n  },\n  async ({\n    tableHandlers,\n    agentHandlers,\n    orchestratorToolHandlers,\n    userInputValues,\n    setProgress,\n  }) => {\n    const toPathArray = (value: unknown): string[] => {\n      if (Array.isArray(value)) {\n        const out: string[] = [];\n        for (const item of value) {\n          if (typeof item === "string" && item.trim()) out.push(item);\n        }\n        return out;\n      }\n      if (typeof value === "string" && value.trim()) return [value];\n      return [];\n    };\n\n    const normalizeDate = (value: string | undefined): string | null => {\n      if (!value) return null;\n      const trimmed = value.trim();\n      if (!trimmed) return null;\n      if (/^\\d{4}-\\d{2}-\\d{2}$/.test(trimmed)) return trimmed;\n      const d = new Date(trimmed);\n      if (Number.isNaN(d.getTime())) return null;\n      return d.toISOString().slice(0, 10);\n    };\n\n    const normalizeAmount = (value: number | undefined): number | null => {\n      if (typeof value !== "number" || !Number.isFinite(value)) return null;\n      return Math.round(value * 100) / 100;\n    };\n\n    const scanPdfFiles = async (inputPaths: string[]): Promise<string[]> => {\n      const found: string[] = [];\n\n      const walk = async (p: string): Promise<void> => {\n        let stat;\n        try {\n          stat = await fs.stat(p);\n        } catch {\n          return;\n        }\n\n        if (stat.isDirectory()) {\n          const entries = await fs.readdir(p, { withFileTypes: true });\n          for (const entry of entries) {\n            const fullPath = path.join(p, entry.name);\n            if (entry.isDirectory()) {\n              await walk(fullPath);\n            } else if (entry.isFile() && entry.name.toLowerCase().endsWith(".pdf")) {\n              found.push(fullPath);\n            }\n          }\n          return;\n        }\n\n        if (stat.isFile() && p.toLowerCase().endsWith(".pdf")) {\n          found.push(p);\n        }\n      };\n\n      for (const p of inputPaths) {\n        await walk(p);\n      }\n\n      return found;\n    };\n\n    const sourcePaths = toPathArray(userInputValues.sourcePaths);\n    if (!sourcePaths.length) {\n      throw new Error("No source path provided. Please select at least one PDF file or folder.");\n    }\n\n    const importMode =\n      userInputValues.importMode === "insert_automatically"\n        ? "insert_automatically"\n        : "preview_then_insert";\n    const duplicateHandling =\n      userInputValues.duplicateHandling === "insert_all" ||\n      userInputValues.duplicateHandling === "update_existing"\n        ? userInputValues.duplicateHandling\n        : "skip";\n    const missingFieldHandling =\n      userInputValues.missingFieldHandling === "infer_best_effort" ||\n      userInputValues.missingFieldHandling === "fail_run"\n        ? userInputValues.missingFieldHandling\n        : "skip_file";\n    const defaultCurrency =\n      typeof userInputValues.defaultCurrency === "string" && userInputValues.defaultCurrency.length === 3\n        ? userInputValues.defaultCurrency\n        : "USD";\n\n    await setProgress(5, "Scanning input paths for PDF receipts...");\n    const pdfFiles = await scanPdfFiles(sourcePaths);\n    if (!pdfFiles.length) {\n      throw new Error("No PDF files found in the selected paths.");\n    }\n\n    await setProgress(10, `Found ${pdfFiles.length} PDF files. Starting extraction...`);\n\n    const previewRows: Record<string, unknown>[] = [];\n    const skippedMissing: { file: string; reason: string }[] = [];\n    const skippedDuplicates: { file: string; duplicateId: unknown }[] = [];\n    const extractionErrors: { file: string; error: string }[] = [];\n    let updatedCount = 0;\n    let insertedCount = 0;\n\n    type ReceiptInsertRow = Parameters<typeof tableHandlers.receipts.insertMany>[0][number];\n    const pendingInsertBatch: ReceiptInsertRow[] = [];\n\n    const flushInsertBatch = async (): Promise<void> => {\n      if (!pendingInsertBatch.length) return;\n      await tableHandlers.receipts.insertMany(pendingInsertBatch);\n      insertedCount += pendingInsertBatch.length;\n      pendingInsertBatch.length = 0;\n    };\n\n    for (const [index, filePath] of pdfFiles.entries()) {\n      const progress = 10 + Math.floor(((index + 1) / pdfFiles.length) * 85);\n      await setProgress(progress, `Processing ${index + 1}/${pdfFiles.length}: ${path.basename(filePath)}`);\n\n      try {\n        const buffer = await fs.readFile(filePath);\n        const fileAsBase64 = buffer.toString("base64");\n\n        const docText = await orchestratorToolHandlers.documents.get_document_text({\n          contentType: "application/pdf",\n          fileAsBase64,\n           do_ocr: true,\n          do_table_structure: true,\n        });\n\n        const extraction = await agentHandlers.receiptExtractor(\n          [\n            `File: ${filePath}`,\n            "Extract the receipt fields from the OCR text below.",\n            "If missing, return null-equivalent by omitting the field.",\n            "OCR TEXT:",\n            docText.slice(0, 40000),\n          ].join("\\n\\n"),\n        );\n\n        const vendorName =\n          typeof extraction.vendor_name === "string" ? extraction.vendor_name.trim() : "";\n        const purchaseDate = normalizeDate(extraction.purchase_date);\n        let subtotal = normalizeAmount(extraction.subtotal);\n        let taxAmount = normalizeAmount(extraction.tax_amount);\n        let totalAmount = normalizeAmount(extraction.total_amount);\n        const receiptNumber =\n          typeof extraction.receipt_number === "string" && extraction.receipt_number.trim()\n            ? extraction.receipt_number.trim()\n            : null;\n        const paymentMethod =\n          typeof extraction.payment_method === "string" && extraction.payment_method.trim()\n            ? extraction.payment_method.trim()\n            : null;\n        const notes =\n          typeof extraction.notes === "string" && extraction.notes.trim()\n            ? extraction.notes.trim()\n            : `Imported from ${filePath}`;\n        const currencyCodeCandidate =\n          typeof extraction.currency_code === "string" && extraction.currency_code.trim().length >= 3\n            ? extraction.currency_code.trim().toUpperCase().slice(0, 3)\n            : defaultCurrency;\n\n        if (missingFieldHandling === "infer_best_effort") {\n          if (totalAmount !== null && taxAmount !== null && subtotal === null) {\n            subtotal = Math.max(0, Math.round((totalAmount - taxAmount) * 100) / 100);\n          }\n          if (subtotal !== null && taxAmount === null && totalAmount !== null) {\n            taxAmount = Math.max(0, Math.round((totalAmount - subtotal) * 100) / 100);\n          }\n          if (subtotal !== null && taxAmount !== null && totalAmount === null) {\n            totalAmount = Math.round((subtotal + taxAmount) * 100) / 100;\n          }\n        }\n\n        const missingRequired =\n          !vendorName || !purchaseDate || subtotal === null || totalAmount === null;\n\n        if (missingRequired) {\n          const reason =\n            "Missing one or more required fields (vendor_name, purchase_date, subtotal, total_amount).";\n\n          if (missingFieldHandling === "fail_run") {\n            throw new Error(`${reason} File: ${filePath}`);\n          }\n\n          skippedMissing.push({ file: filePath, reason });\n          continue;\n        }\n\n        const subtotalValue = subtotal;\n        const totalAmountValue = totalAmount;\n        const taxAmountValue = taxAmount ?? 0;\n\n        if (\n          subtotalValue === null ||\n          totalAmountValue === null ||\n          subtotalValue < 0 ||\n          taxAmountValue < 0 ||\n          totalAmountValue < 0 ||\n          totalAmountValue < subtotalValue\n        ) {\n          skippedMissing.push({\n            file: filePath,\n            reason: "Amounts failed table constraints (non-negative and total >= subtotal).",\n          });\n          continue;\n        }\n\n        const duplicateFilter: Record<string, unknown> = {\n          vendor_name: vendorName,\n          purchase_date: purchaseDate,\n          total_amount: totalAmountValue,\n          receipt_number: receiptNumber,\n        };\n\n        const existing = await tableHandlers.receipts.find(duplicateFilter, {\n          limit: 1,\n          select: { id: 1 },\n        });\n\n        const row: ReceiptInsertRow = {\n          vendor_name: vendorName,\n          purchase_date: purchaseDate,\n          currency_code: currencyCodeCandidate,\n          subtotal: subtotalValue,\n          tax_amount: taxAmountValue,\n          total_amount: totalAmountValue,\n          receipt_number: receiptNumber,\n      /*        payment_method: paymentMethod,\n        notes,\n        metadata: {\n            source_file: filePath,\n            extractor_confidence:\n              typeof extraction.confidence === "number" ? extraction.confidence : null,\n            imported_via_workflow: true,\n          },\n  */      };\n\n        previewRows.push({\n          file: filePath,\n          ...row,\n          duplicate_found: existing.length > 0,\n        });\n\n        if (importMode !== "insert_automatically") {\n          continue;\n        }\n\n        if (existing.length > 0 && duplicateHandling === "skip") {\n          skippedDuplicates.push({ file: filePath, duplicateId: existing[0]?.id });\n          continue;\n        }\n\n        if (existing.length > 0 && duplicateHandling === "update_existing") {\n          const existingId = existing[0]?.id;\n          if (existingId !== undefined && existingId !== null) {\n            await tableHandlers.receipts.update({ id: existingId }, row, { multi: false });\n            updatedCount += 1;\n            continue;\n          }\n        }\n\n        pendingInsertBatch.push(row);\n        if (pendingInsertBatch.length >= 10) {\n          await flushInsertBatch();\n        }\n      } catch (error) {\n        const message = error instanceof Error ? error.message : String(error);\n        extractionErrors.push({ file: filePath, error: message });\n      }\n    }\n\n    if (importMode === "insert_automatically") {\n      await flushInsertBatch();\n    }\n\n    await setProgress(98, "Finalizing import summary...");\n\n    console.log("=== Receipt Import Summary ===");\n    console.log(`Files scanned: ${pdfFiles.length}`);\n    console.log(`Rows extracted: ${previewRows.length}`);\n    console.log(`Skipped (missing/invalid): ${skippedMissing.length}`);\n    console.log(`Skipped (duplicates): ${skippedDuplicates.length}`);\n    console.log(`Extraction errors: ${extractionErrors.length}`);\n\n    if (importMode === "insert_automatically") {\n      console.log(`Inserted rows: ${insertedCount}`);\n      console.log(`Updated rows: ${updatedCount}`);\n    } else {\n      console.log(\n        "Preview mode: no DB writes were made. Re-run with importMode=insert_automatically to commit.",\n      );\n    }\n\n    if (previewRows.length) {\n      const previewSample = previewRows.slice(0, 20);\n      console.log("Preview sample (first 20 rows):");\n      console.log(JSON.stringify(previewSample, null, 2));\n    }\n\n    if (skippedMissing.length) {\n      console.log("Skipped files (missing/invalid required fields):");\n      console.log(JSON.stringify(skippedMissing.slice(0, 50), null, 2));\n    }\n\n    if (skippedDuplicates.length) {\n      console.log("Skipped duplicates:");\n      console.log(JSON.stringify(skippedDuplicates.slice(0, 50), null, 2));\n    }\n\n    if (extractionErrors.length) {\n      console.log("Files with extraction/runtime errors:");\n      console.log(JSON.stringify(extractionErrors.slice(0, 50), null, 2));\n    }\n\n    await setProgress(100, "Receipt import workflow completed.");\n  },\n);\n',
+          workflow_function_definition_summary:
+            "Imports PDF receipts from selected files/folders into the existing receipts table. It OCRs each PDF using the documents tool, extracts structured fields via an agent, checks duplicates (vendor+date+total+receipt_number), and supports two run modes: preview-only (default) or automatic insert. It also supports configurable duplicate and missing-field handling, provides progress updates, uses batched insertMany writes, and prints a clear summary with skipped/error files.",
+        }),
+      ],
+    },
+  ],
+};

package/e2e/tests/testAskLLM/speechToTextTest.ts

@@ -0,0 +1,101 @@
+import { expect } from "@playwright/test";
+import { readFileSync } from "fs";
+import { join } from "path";
+import { getDataKey } from "Testing";
+import { newChat, type PageWIds } from "utils/utils";
+
+export const speechToTextTest = async (page: PageWIds) => {
+  await grantMicrophonePermission(page);
+  await mockMediaDevicesWithAudioFile(page);
+  await page.reload();
+  await page.getByTestId("AskLLM").click();
+
+  await newChat(page);
+  await page.getByTestId("Chat.speech").click();
+  await page.locator(getDataKey("stt-local")).click();
+  await page
+    .locator(getDataKey("speechToText") + " " + getDataKey("model"))
+    .click();
+  await page.locator(getDataKey("tiny")).click();
+  await page
+    .locator(getDataKey("speechToText") + " " + getDataKey("service-toggle"))
+    .click();
+  await expect(
+    page.locator(
+      getDataKey("speechToText") + " " + getDataKey("service-status"),
+    ),
+    /** Flaky. gets stuck on "starting". Due to desktop/ui clash? */
+  ).toContainText("running", { timeout: 5 * 60_000 });
+
+  // Used to debug
+  await page.locator(getDataKey("audio")).click();
+  await page.getByTestId("Popup.close").last().click();
+
+  await page.getByTestId("Chat.speech").click();
+  await page.waitForTimeout(2000);
+  await page
+    .getByTestId("Chat.sendWrapper")
+    .locator("audio")
+    .waitFor({ state: "visible" });
+  await page.getByTestId("Chat.send").click();
+  await page
+    .getByTestId("Chat.messageList")
+    .locator("audio")
+    .waitFor({ state: "visible" });
+
+  await page
+    .getByTestId("Chat.speech")
+    .click({ button: "right", timeout: 10_000 });
+  await page.locator(getDataKey("stt-local")).click();
+  await page.getByTestId("Popup.close").last().click();
+
+  await page.getByTestId("Chat.speech").click();
+  await page.waitForTimeout(2000);
+  await page.waitForTimeout(2000);
+  await expect(page.getByTestId("Chat.textarea")).toHaveValue("Hello.", {
+    timeout: 10000,
+  });
+};
+
+// Helper to grant microphone permissions
+const grantMicrophonePermission = async (page: PageWIds): Promise<void> => {
+  await page.context().grantPermissions(["microphone"]);
+};
+const mockMediaDevicesWithAudioFile = async (page: PageWIds): Promise<void> => {
+  // Read file in Node.js context BEFORE injecting
+  const buffer = readFileSync(join(__dirname, "hello.mp3"));
+  const base64Audio = buffer.toString("base64");
+
+  await page.addInitScript((audioBase64: string) => {
+    navigator.mediaDevices.getUserMedia = async (constraints) => {
+      console.log("[Mock] getUserMedia called with:", constraints);
+      // simulate real user delay
+      await new Promise((resolve) => setTimeout(resolve, 2000));
+      if (constraints?.audio) {
+        // Decode base64 to ArrayBuffer in browser
+        const binaryString = atob(audioBase64);
+        const bytes = new Uint8Array(binaryString.length);
+        for (let i = 0; i < binaryString.length; i++) {
+          bytes[i] = binaryString.charCodeAt(i);
+        }
+
+        const audioContext = new AudioContext();
+        if (audioContext.state === "suspended") {
+          await audioContext.resume();
+        }
+        const audioBuffer = await audioContext.decodeAudioData(bytes.buffer);
+
+        const source = audioContext.createBufferSource();
+        const destination = audioContext.createMediaStreamDestination();
+
+        source.buffer = audioBuffer;
+        source.connect(destination);
+        source.start();
+
+        console.log("[Mock] Returning mock stream");
+        return destination.stream;
+      }
+      throw new Error("No audio constraints");
+    };
+  }, base64Audio);
+};

package/e2e/tests/testAskLLM/stringify.ts

@@ -0,0 +1 @@
+export const stringify = (obj: any) => JSON.stringify(obj, null, 2);