@talkpilot/core-db 1.3.3 → 1.3.5

package/src/municipal/muniIssues/muniIssues.constants.ts

@@ -0,0 +1,18 @@
+/** Mongo collection name for reported municipal issues. */
+export const MUNI_ISSUES_COLLECTION = "muniIssues";
+
+/** Sync state of the issue's Jira ticket at write time. */
+export const MUNI_ISSUE_SYNC_STATUSES = ["created", "failed"] as const;
+
+/** Max length for the long free-text fields — well above the ~2000-char target. */
+export const MUNI_ISSUE_TEXT_MAX_LENGTH = 5000;
+
+/** Max length for the short identity fields (a person's name and role). */
+export const MUNI_ISSUE_NAME_MAX_LENGTH = 200;
+
+/** MongoDB error 48 (NamespaceExists): collection already exists, e.g. concurrent createCollection. */
+export const NAMESPACE_EXISTS = 48;
+
+/** Reject (not just flag) writes that fail the $jsonSchema validator. */
+export const VALIDATION_LEVEL = "strict" as const;
+export const VALIDATION_ACTION = "error" as const;

package/src/municipal/muniIssues/muniIssues.getters.ts

@@ -0,0 +1,24 @@
+import { Collection, ObjectId } from "mongodb";
+import { getDb } from "../index";
+import { MuniIssue } from "./muniIssues.types";
+import { MUNI_ISSUES_COLLECTION } from "./muniIssues.constants";
+
+export const getMuniIssuesCollection = (): Collection<MuniIssue> => {
+  return getDb().collection<MuniIssue>(MUNI_ISSUES_COLLECTION);
+};
+
+export const generateMuniIssueId = (): ObjectId => new ObjectId();
+
+export const getMuniIssueById = async (
+  id: string,
+): Promise<MuniIssue | null> => {
+  // A malformed id is treated as not-found rather than throwing on `new ObjectId`.
+  if (!ObjectId.isValid(id)) return null;
+  return getMuniIssuesCollection().findOne({ _id: new ObjectId(id) });
+};
+
+export const getMuniIssueByJiraKey = async (
+  jiraKey: string,
+): Promise<MuniIssue | null> => {
+  return getMuniIssuesCollection().findOne({ "jira.key": jiraKey });
+};

package/src/municipal/muniIssues/muniIssues.schema.ts

@@ -0,0 +1,66 @@
+import type { Document, IndexDescription } from "mongodb";
+import {
+  MUNI_ISSUE_SYNC_STATUSES,
+  MUNI_ISSUE_TEXT_MAX_LENGTH,
+  MUNI_ISSUE_NAME_MAX_LENGTH,
+} from "./muniIssues.constants";
+
+export const muniIssuesJsonSchema: Document = {
+  bsonType: "object",
+  required: [
+    "callSid",
+    "clientId",
+    "cityName",
+    "issueContent",
+    "jira",
+    "createdAt",
+  ],
+  properties: {
+    callSid: { bsonType: "string" },
+    clientId: { bsonType: "string" },
+    cityName: { bsonType: "string" },
+    issueContent: {
+      bsonType: "object",
+      required: [
+        "issuerName",
+        "issuerRole",
+        "issueDescription",
+        "desiredBehavior",
+      ],
+      properties: {
+        issuerName: {
+          bsonType: "string",
+          maxLength: MUNI_ISSUE_NAME_MAX_LENGTH,
+        },
+        issuerRole: {
+          bsonType: "string",
+          maxLength: MUNI_ISSUE_NAME_MAX_LENGTH,
+        },
+        issueDescription: {
+          bsonType: "string",
+          maxLength: MUNI_ISSUE_TEXT_MAX_LENGTH,
+        },
+        desiredBehavior: {
+          bsonType: "string",
+          maxLength: MUNI_ISSUE_TEXT_MAX_LENGTH,
+        },
+      },
+    },
+    jira: {
+      bsonType: "object",
+      required: ["key", "syncStatus"],
+      properties: {
+        key: { bsonType: "string" },
+        syncStatus: { enum: [...MUNI_ISSUE_SYNC_STATUSES] },
+      },
+    },
+    createdAt: { bsonType: "date" },
+  },
+};
+
+export const muniIssuesIndexes: IndexDescription[] = [
+  { key: { callSid: 1 }, name: "callSid_1" },
+  { key: { createdAt: -1 }, name: "createdAt_-1" },
+  // Unique: one stored document per Jira ticket, guarding against duplicate writes.
+  { key: { "jira.key": 1 }, name: "jira_key_unique", unique: true },
+];

package/src/municipal/muniIssues/muniIssues.setters.ts

@@ -0,0 +1,24 @@
+import { ObjectId } from "mongodb";
+import { CreateMuniIssueInput, MuniIssue } from "./muniIssues.types";
+import { getMuniIssuesCollection } from "./muniIssues.getters";
+import { ensureMuniIssuesCollectionOnce } from "./muniIssues.setup";
+
+// Required `id`: MIS pre-generates it (generateMuniIssueId) and embeds it in the Jira ticket — the two-way link.
+export const createMuniIssue = async (
+  issueData: CreateMuniIssueInput,
+  id: ObjectId,
+): Promise<ObjectId> => {
+  await ensureMuniIssuesCollectionOnce();
+
+  const issue: MuniIssue = {
+    _id: id,
+    ...issueData,
+    jira: {
+      ...issueData.jira,
+      syncStatus: issueData.jira.syncStatus ?? "created",
+    },
+    createdAt: new Date(),
+  };
+  const { insertedId } = await getMuniIssuesCollection().insertOne(issue);
+  return insertedId;
+};

package/src/municipal/muniIssues/muniIssues.setup.ts

@@ -0,0 +1,65 @@
+import { Db, Document, MongoServerError } from "mongodb";
+import { getDb } from "../index";
+import {
+  MUNI_ISSUES_COLLECTION,
+  NAMESPACE_EXISTS,
+  VALIDATION_LEVEL,
+  VALIDATION_ACTION,
+} from "./muniIssues.constants";
+import { muniIssuesIndexes, muniIssuesJsonSchema } from "./muniIssues.schema";
+
+const applyValidator = (db: Db, validator: Document): Promise<unknown> =>
+  db.command({
+    collMod: MUNI_ISSUES_COLLECTION,
+    validator,
+    validationLevel: VALIDATION_LEVEL,
+    validationAction: VALIDATION_ACTION,
+  });
+
+// Provisions validator and indexes: createCollection first, collMod if already exists.
+export const ensureMuniIssuesCollection = async (): Promise<void> => {
+  const db = getDb();
+  const validator = { $jsonSchema: muniIssuesJsonSchema };
+
+  const existing = await db
+    .listCollections({ name: MUNI_ISSUES_COLLECTION })
+    .toArray();
+
+  if (existing.length === 0) {
+    try {
+      await db.createCollection(MUNI_ISSUES_COLLECTION, {
+        validator,
+        validationLevel: VALIDATION_LEVEL,
+        validationAction: VALIDATION_ACTION,
+      });
+    } catch (err) {
+      // NamespaceExists race: collection exists — apply validator via collMod.
+      if (err instanceof MongoServerError && err.code === NAMESPACE_EXISTS) {
+        await applyValidator(db, validator);
+      } else {
+        throw err;
+      }
+    }
+  } else {
+    await applyValidator(db, validator);
+  }
+
+  await db.collection(MUNI_ISSUES_COLLECTION).createIndexes(muniIssuesIndexes);
+};
+
+// Per-Db cache so each test's in-memory Db re-provisions.
+const provisioningByDb = new WeakMap<Db, Promise<void>>();
+
+// Once per Db; evicts failed run so the next call retries.
+export const ensureMuniIssuesCollectionOnce = (): Promise<void> => {
+  const db = getDb();
+  let provisioning = provisioningByDb.get(db);
+  if (!provisioning) {
+    provisioning = ensureMuniIssuesCollection().catch((err) => {
+      provisioningByDb.delete(db);
+      throw err;
+    });
+    provisioningByDb.set(db, provisioning);
+  }
+  return provisioning;
+};

package/src/municipal/muniIssues/muniIssues.types.ts

@@ -0,0 +1,37 @@
+import { ObjectId } from "mongodb";
+
+import { CityName } from "../utils/types";
+import { MUNI_ISSUE_SYNC_STATUSES } from "./muniIssues.constants";
+
+export type MuniIssueSyncStatus = (typeof MUNI_ISSUE_SYNC_STATUSES)[number];
+
+export type MuniIssueContent = {
+  issuerName: string;
+  issuerRole: string;
+  issueDescription: string;
+  desiredBehavior: string;
+};
+
+export type MuniIssueJira = {
+  key: string;
+  syncStatus: MuniIssueSyncStatus;
+};
+
+export type MuniIssue = {
+  _id: ObjectId;
+  callSid: string;
+  clientId: string;
+  cityName: CityName;
+  issueContent: MuniIssueContent;
+  jira: MuniIssueJira;
+  createdAt: Date;
+};
+
+// _id/createdAt are set on insert; jira.syncStatus defaults to "created"
+// because the issue is only stored after its Jira ticket exists (CP-1115).
+export type CreateMuniIssueInput = Omit<
+  MuniIssue,
+  "_id" | "createdAt" | "jira"
+> & {
+  jira: { key: string; syncStatus?: MuniIssueSyncStatus };
+};

package/src/municipal/tickets/__tests__/tickets.getters.spec.ts

@@ -1,7 +1,4 @@
-import {
-  findTicketsByCallSid,
-  getTicketsCountByCityAndDateRange,
-} from "../tickets.getters";
+import { findTicketsByCallSid } from "../tickets.getters";
 import { setDb } from "../../index";
 import { Db, Collection } from "mongodb";
 import { createTicket } from "../../../test-utils/factories";
@@ -30,37 +27,4 @@ describe("tickets getters", () => {
     expect(mockCollection.find).toHaveBeenCalledWith({ callSid: "123" });
     expect(result[0].callSid).toBe("123");
   });
-
-  it("getTicketsCountByCityAndDateRange should not count tickets without callSid", async () => {
-    const aggregateNext = jest.fn().mockResolvedValue({ n: 1 });
-    mockCollection = {
-      ...mockCollection,
-      aggregate: jest.fn().mockReturnValue({
-        next: aggregateNext,
-      }),
-    };
-    mockDb = {
-      collection: jest.fn().mockReturnValue(mockCollection),
-    };
-    setDb(mockDb as Db);
-
-    const result = await getTicketsCountByCityAndDateRange(
-      "ashdod",
-      "2026-01-01",
-      "2026-01-31",
-      "Asia/Jerusalem",
-    );
-
-    expect(mockCollection.aggregate).toHaveBeenCalledWith(
-      expect.arrayContaining([
-        {
-          $match: {
-            cityName: "ashdod",
-            callSid: { $exists: true, $nin: [null, ""] },
-          },
-        },
-      ]),
-    );
-    expect(result).toBe(1);
-  });
 });

package/src/municipal/tickets/__tests__/tickets.statistics.spec.ts

@@ -0,0 +1,51 @@
+import {
+  findCallSidsWithDraftTicketsByCity,
+  findCallSidsWithTicketsByCity,
+  getTicketsCollection,
+  ticketStatsDateScopeFromYmd,
+} from "../../../index";
+import { createTicket } from "../../../test-utils/factories";
+
+const CITY = "tests" as const;
+const TZ = "Asia/Jerusalem";
+const mayScope = () => ticketStatsDateScopeFromYmd("2026-05-01", "2026-05-31", TZ);
+
+describe("tickets statistics getters", () => {
+  it("should scope callSid lookups to the provided createdAt range", async () => {
+    await getTicketsCollection().insertMany([
+      createTicket({
+        cityName: CITY,
+        callSid: "CA-in-range",
+        createdAt: new Date("2026-05-15T10:00:00.000Z"),
+        updatedAt: new Date("2026-05-15T10:00:00.000Z"),
+      }),
+      createTicket({
+        cityName: CITY,
+        callSid: "CA-out-of-range",
+        createdAt: new Date("2024-01-01T10:00:00.000Z"),
+        updatedAt: new Date("2024-01-01T10:00:00.000Z"),
+      }),
+    ]);
+    expect(await findCallSidsWithTicketsByCity(CITY, mayScope())).toEqual(["CA-in-range"]);
+  });
+
+  it("should treat whitespace-only event_subject_id as draft", async () => {
+    await getTicketsCollection().insertMany([
+      createTicket({
+        cityName: CITY,
+        callSid: "CA-draft-whitespace",
+        createdAt: new Date("2026-05-15T10:00:00.000Z"),
+        updatedAt: new Date("2026-05-15T10:00:00.000Z"),
+        externalCallFields: { event_subject_id: "  \t  " },
+      }),
+      createTicket({
+        cityName: CITY,
+        callSid: "CA-not-draft",
+        createdAt: new Date("2026-05-16T10:00:00.000Z"),
+        updatedAt: new Date("2026-05-16T10:00:00.000Z"),
+        externalCallFields: { event_subject_id: "100" },
+      }),
+    ]);
+    expect(await findCallSidsWithDraftTicketsByCity(CITY, mayScope())).toEqual(["CA-draft-whitespace"]);
+  });
+});

package/src/municipal/tickets/index.ts

@@ -1,2 +1,3 @@
 export * from "./tickets.getters";
-export type { Ticket, TicketDoc, SubjectStatsItem } from "./tickets.types";
+export * from "./tickets.statistics.getters";
+export type { Ticket, TicketDoc, SubjectItem, TicketStatsDateScope, TicketSubjectRow } from "./tickets.types";

package/src/municipal/tickets/tickets.constants.ts

@@ -0,0 +1,8 @@
+/** Label used when a ticket has no resolvable department/subject. */
+export const UNCLASSIFIED = "ללא מחלקה";
+
+/** Max execution time (ms) for ticket statistics aggregations. */
+export const STATS_MAX_TIME_MS = 30_000;
+
+/** Default look-back window (days) for ticket statistics date ranges. */
+export const DEFAULT_LOOKBACK_DAYS = 30;

package/src/municipal/tickets/tickets.getters.ts

@@ -1,5 +1,4 @@
 import { CityName, getDb, ObjectId, Ticket } from "../index";
-import type { SubjectStatsItem } from "./tickets.types";
 import { Collection, Filter, ObjectId as MongoObjectId } from "mongodb";
 
 /**
@@ -68,145 +67,6 @@ export const deleteTicket = async (ticketId: string): Promise<boolean> => {
   return result.deletedCount > 0;
 };
 
-/**
- * Count tickets by city and date range (createdAt converted to given timezone).
- * Used as "open" tickets count when status is not available.
- */
-export async function getTicketsCountByCityAndDateRange(
-  cityName: string,
-  startStr: string,
-  endStr: string,
-  timezone: string,
-): Promise<number> {
-  const doc = await getTicketsCollection()
-    .aggregate<{ n: number }>([
-      { $match: { cityName, callSid: { $exists: true, $nin: [null, ""] } } },
-      {
-        $addFields: {
-          dateLocal: {
-            $dateToString: { format: "%Y-%m-%d", date: "$createdAt", timezone },
-          },
-        },
-      },
-      { $match: { dateLocal: { $gte: startStr, $lte: endStr } } },
-      { $count: "n" },
-    ])
-    .next();
-  return doc?.n ?? 0;
-}
-
-/**
- * Count of tickets by department (subject) from departmentsSubjects. Different departments (subject_id) are grouped, and sub-subjects are unified under their department.
- * Date filter: createdAt in [startStr, endStr] (in the given timezone). Fallback to "Unclassified" when no match is found in the lookup.
- */
-export async function getTicketsSubjectStats(
-  cityName: string,
-  startStr: string,
-  endStr: string,
-  timezone: string,
-): Promise<SubjectStatsItem[]> {
-  const coll = getTicketsCollection();
-  const rows = await coll
-    .aggregate<{ _id: string; subject: string; count: number }>([
-      { $match: { cityName } },
-      {
-        $addFields: {
-          dateLocal: {
-            $dateToString: { format: "%Y-%m-%d", date: "$createdAt", timezone },
-          },
-        },
-      },
-      { $match: { dateLocal: { $gte: startStr, $lte: endStr } } },
-      {
-        $addFields: {
-          effectiveSubjectId: {
-            $ifNull: [
-              "$externalCallFields.event_subject_id",
-              {
-                $ifNull: [
-                  "$externalCallFields.event_sub_subject_id",
-                  "$externalCallFields.event_sub_subject_id2",
-                ],
-              },
-            ],
-          },
-        },
-      },
-      {
-        $lookup: {
-          from: "departmentsSubjects",
-          let: { sid: "$effectiveSubjectId", c: cityName },
-          pipeline: [
-            {
-              $match: {
-                $expr: {
-                  $and: [
-                    { $eq: ["$cityName", "$$c"] },
-                    {
-                      $or: [
-                        { $eq: ["$subject_id", { $ifNull: ["$$sid", ""] }] },
-                        {
-                          $eq: ["$sub_subject_id", { $ifNull: ["$$sid", ""] }],
-                        },
-                        {
-                          $eq: ["$sub_subject_id2", { $ifNull: ["$$sid", ""] }],
-                        },
-                      ],
-                    },
-                  ],
-                },
-              },
-            },
-            { $limit: 1 },
-          ],
-          as: "subj",
-        },
-      },
-      {
-        $addFields: {
-          subject_id: {
-            $cond: {
-              if: { $gt: [{ $size: "$subj" }, 0] },
-              then: {
-                $ifNull: [{ $arrayElemAt: ["$subj.subject_id", 0] }, ""],
-              },
-              else: "",
-            },
-          },
-          subject: {
-            $cond: {
-              if: { $gt: [{ $size: "$subj" }, 0] },
-              then: {
-                $ifNull: [
-                  { $arrayElemAt: ["$subj.subjectName", 0] },
-                  "Unclassified",
-                ],
-              },
-              else: "Unclassified",
-            },
-          },
-        },
-      },
-      {
-        $group: {
-          _id: "$subject_id",
-          subject: { $first: "$subject" },
-          count: { $sum: 1 },
-        },
-      },
-      { $sort: { count: -1 } },
-    ])
-    .toArray();
-
-  const total = rows.reduce((s: number, r: any) => s + r.count, 0);
-  return rows.map((r: any) => ({
-    subject_name: r.subject,
-    subject_id: r._id,
-    count: r.count,
-    percentage: total > 0 ? Math.round((r.count / total) * 100) : 0,
-  }));
-}
-
 export const findTicketByQuery = async (query: Partial<Ticket>) => {
   return await getTicketsCollection().findOne(query);
 };

package/src/municipal/tickets/tickets.statistics.aggregation.ts

@@ -0,0 +1,96 @@
+import { CityName } from "../utils/types";
+import { STATS_MAX_TIME_MS, UNCLASSIFIED } from "./tickets.constants";
+import { getTicketsCollection } from "./tickets.getters";
+import type { TicketStatsDateRange } from "./tickets.types";
+
+const effectiveSubjectIdExpr = {
+  $ifNull: [
+    "$externalCallFields.event_subject_id",
+    {
+      $ifNull: [
+        "$externalCallFields.event_sub_subject_id",
+        "$externalCallFields.event_sub_subject_id2",
+      ],
+    },
+  ],
+};
+
+const departmentsSubjectsLookup = (cityName: CityName) => ({
+  $lookup: {
+    from: "departmentsSubjects",
+    let: { sid: "$effectiveSubjectId", c: cityName },
+    pipeline: [
+      {
+        $match: {
+          $expr: {
+            $and: [
+              { $eq: ["$cityName", "$$c"] },
+              {
+                $or: [
+                  { $eq: ["$subject_id", { $ifNull: ["$$sid", ""] }] },
+                  { $eq: ["$sub_subject_id", { $ifNull: ["$$sid", ""] }] },
+                  { $eq: ["$sub_subject_id2", { $ifNull: ["$$sid", ""] }] },
+                ],
+              },
+            ],
+          },
+        },
+      },
+      { $limit: 1 },
+    ],
+    as: "subj",
+  },
+});
+
+const subjectLabelExpr = {
+  $cond: {
+    if: { $gt: [{ $size: "$subj" }, 0] },
+    then: {
+      $ifNull: [{ $arrayElemAt: ["$subj.subjectName", 0] }, UNCLASSIFIED],
+    },
+    else: UNCLASSIFIED,
+  },
+};
+
+const ticketDateMatch = (dateRange: TicketStatsDateRange) => ({
+  createdAt: { $gte: dateRange.from, $lte: dateRange.to },
+});
+
+export const isDraftSubjectIdExpr = {
+  $or: [
+    { $eq: [{ $type: "$externalCallFields.event_subject_id" }, "missing"] },
+    { $eq: ["$externalCallFields.event_subject_id", null] },
+    {
+      $regexMatch: {
+        input: { $ifNull: ["$externalCallFields.event_subject_id", ""] },
+        regex: /^\s*$/,
+      },
+    },
+  ],
+};
+
+export const ticketsWithCallSidMatch = (
+  cityName: CityName,
+  dateRange: TicketStatsDateRange,
+) => ({
+  cityName,
+  callSid: { $exists: true, $nin: [null, ""] },
+  ...ticketDateMatch(dateRange),
+});
+
+export const subjectResolutionStages = (
+  cityName: CityName,
+): Record<string, unknown>[] => [
+  { $addFields: { effectiveSubjectId: effectiveSubjectIdExpr } },
+  departmentsSubjectsLookup(cityName),
+  { $addFields: { subject: subjectLabelExpr } },
+];
+
+export const runTicketStatsAggregate = async <T>(
+  pipeline: Record<string, unknown>[],
+): Promise<T[]> => {
+  const rows = await getTicketsCollection()
+    .aggregate(pipeline, { maxTimeMS: STATS_MAX_TIME_MS })
+    .toArray();
+  return rows as T[];
+};

package/src/municipal/tickets/tickets.statistics.getters.ts

@@ -0,0 +1,71 @@
+import { CityName } from "../utils/types";
+import {
+  DAY_MS,
+  endOfCalendarDayInTz,
+  startOfCalendarDayInTz,
+} from "../../utils/date.utils";
+import { DEFAULT_LOOKBACK_DAYS } from "./tickets.constants";
+import {
+  isDraftSubjectIdExpr,
+  runTicketStatsAggregate,
+  subjectResolutionStages,
+  ticketsWithCallSidMatch,
+} from "./tickets.statistics.aggregation";
+import type {
+  TicketStatsDateRange,
+  TicketStatsDateScope,
+  TicketSubjectRow,
+} from "./tickets.types";
+
+export const resolveTicketStatsDateRange = (
+  dateScope?: TicketStatsDateScope,
+): TicketStatsDateRange => {
+  const to = dateScope?.to ?? new Date();
+  const from =
+    dateScope?.from ?? new Date(to.getTime() - DEFAULT_LOOKBACK_DAYS * DAY_MS);
+  return { from, to };
+};
+
+export const ticketStatsDateScopeFromYmd = (
+  startStr: string,
+  endStr: string,
+  timezone: string,
+): TicketStatsDateScope => ({
+  from: startOfCalendarDayInTz(startStr, timezone),
+  to: endOfCalendarDayInTz(endStr, timezone),
+});
+
+export const findCallSidsWithTicketsByCity = async (
+  cityName: CityName,
+  dateScope?: TicketStatsDateScope,
+): Promise<string[]> => {
+  const rows = await runTicketStatsAggregate<{ _id: unknown }>([
+    { $match: ticketsWithCallSidMatch(cityName, resolveTicketStatsDateRange(dateScope)) },
+    { $group: { _id: "$callSid" } },
+  ]);
+  return rows.map((r) => String(r._id)).filter(Boolean);
+};
+
+export const findCallSidsWithDraftTicketsByCity = async (
+  cityName: CityName,
+  dateScope?: TicketStatsDateScope,
+): Promise<string[]> => {
+  const rows = await runTicketStatsAggregate<{ _id: unknown }>([
+    { $match: ticketsWithCallSidMatch(cityName, resolveTicketStatsDateRange(dateScope)) },
+    { $match: { $expr: isDraftSubjectIdExpr } },
+    { $group: { _id: "$callSid" } },
+  ]);
+  return rows.map((r) => String(r._id)).filter(Boolean);
+};
+
+export const findTicketSubjectRows = async (
+  cityName: CityName,
+  dateScope?: TicketStatsDateScope,
+): Promise<TicketSubjectRow[]> => {
+  const rows = await runTicketStatsAggregate<{ callSid: unknown; subject: unknown }>([
+    { $match: ticketsWithCallSidMatch(cityName, resolveTicketStatsDateRange(dateScope)) },
+    ...subjectResolutionStages(cityName),
+    { $project: { _id: 0, callSid: 1, subject: 1 } },
+  ]);
+  return rows.map((r) => ({ callSid: String(r.callSid), subject: String(r.subject) }));
+};