@talkpilot/core-db 1.2.2 → 1.3.0

package/src/municipal/tickets/__tests__/tickets.statistics.spec.ts

@@ -0,0 +1,104 @@
+import {
+  findCallSidsWithDraftTicketsByCity,
+  findCallSidsWithTicketsByCity,
+  findSubjectsByCityAndDateRange,
+  findSubjectsByCallSids,
+  getDepartmentsSubjectsCollection,
+  getTicketsCollection,
+  ticketStatsDateScopeFromYmd,
+} from "../../../index";
+import { createDepartmentSubject, createTicket } from "../../../test-utils/factories";
+
+const CITY = "tests" as const;
+const TZ = "Asia/Jerusalem";
+const mayScope = () => ticketStatsDateScopeFromYmd("2026-05-01", "2026-05-31", TZ);
+
+describe("tickets statistics getters", () => {
+  it("should scope callSid lookups to the provided createdAt range", async () => {
+    await getTicketsCollection().insertMany([
+      createTicket({
+        cityName: CITY,
+        callSid: "CA-in-range",
+        createdAt: new Date("2026-05-15T10:00:00.000Z"),
+        updatedAt: new Date("2026-05-15T10:00:00.000Z"),
+      }),
+      createTicket({
+        cityName: CITY,
+        callSid: "CA-out-of-range",
+        createdAt: new Date("2024-01-01T10:00:00.000Z"),
+        updatedAt: new Date("2024-01-01T10:00:00.000Z"),
+      }),
+    ]);
+    expect(await findCallSidsWithTicketsByCity(CITY, mayScope())).toEqual(["CA-in-range"]);
+  });
+
+  it("should treat whitespace-only event_subject_id as draft", async () => {
+    await getTicketsCollection().insertMany([
+      createTicket({
+        cityName: CITY,
+        callSid: "CA-draft-whitespace",
+        createdAt: new Date("2026-05-15T10:00:00.000Z"),
+        updatedAt: new Date("2026-05-15T10:00:00.000Z"),
+        externalCallFields: { event_subject_id: "  \t  " },
+      }),
+      createTicket({
+        cityName: CITY,
+        callSid: "CA-not-draft",
+        createdAt: new Date("2026-05-16T10:00:00.000Z"),
+        updatedAt: new Date("2026-05-16T10:00:00.000Z"),
+        externalCallFields: { event_subject_id: "100" },
+      }),
+    ]);
+    expect(await findCallSidsWithDraftTicketsByCity(CITY, mayScope())).toEqual(["CA-draft-whitespace"]);
+  });
+
+  it("should aggregate subjects by city and createdAt date scope", async () => {
+    await getDepartmentsSubjectsCollection().insertOne(
+      createDepartmentSubject({ cityName: CITY, subject_id: "200", subjectName: "Roads" }),
+    );
+    await getTicketsCollection().insertMany([
+      createTicket({
+        cityName: CITY,
+        callSid: "CA-subject-in-range",
+        createdAt: new Date("2026-05-15T10:00:00.000Z"),
+        updatedAt: new Date("2026-05-15T10:00:00.000Z"),
+        externalCallFields: { event_subject_id: "200" },
+      }),
+      createTicket({
+        cityName: CITY,
+        callSid: "CA-subject-out-of-range",
+        createdAt: new Date("2024-01-01T10:00:00.000Z"),
+        updatedAt: new Date("2024-01-01T10:00:00.000Z"),
+        externalCallFields: { event_subject_id: "200" },
+      }),
+    ]);
+    expect(await findSubjectsByCityAndDateRange(CITY, mayScope())).toEqual([
+      { subject: "Roads", count: 1 },
+    ]);
+  });
+
+  it("should exclude tickets outside dateScope when aggregating subjects by callSid", async () => {
+    await getDepartmentsSubjectsCollection().insertOne(
+      createDepartmentSubject({ cityName: CITY, subject_id: "300", subjectName: "Parks" }),
+    );
+    await getTicketsCollection().insertMany([
+      createTicket({
+        cityName: CITY,
+        callSid: "CA-shared-sid",
+        createdAt: new Date("2026-05-15T10:00:00.000Z"),
+        updatedAt: new Date("2026-05-15T10:00:00.000Z"),
+        externalCallFields: { event_subject_id: "300" },
+      }),
+      createTicket({
+        cityName: CITY,
+        callSid: "CA-shared-sid",
+        createdAt: new Date("2024-01-01T10:00:00.000Z"),
+        updatedAt: new Date("2024-01-01T10:00:00.000Z"),
+        externalCallFields: { event_subject_id: "300" },
+      }),
+    ]);
+    expect(await findSubjectsByCallSids(CITY, ["CA-shared-sid"], undefined, mayScope())).toEqual([
+      { subject: "Parks", count: 1 },
+    ]);
+  });
+});

package/src/municipal/tickets/index.ts

@@ -1,2 +1,3 @@
 export * from "./tickets.getters";
-export type { Ticket, TicketDoc, SubjectStatsItem } from "./tickets.types";
+export * from "./tickets.statistics.getters";
+export type { Ticket, TicketDoc, SubjectItem, TicketStatsDateScope } from "./tickets.types";

package/src/municipal/tickets/tickets.constants.ts

@@ -0,0 +1,8 @@
+/** Label used when a ticket has no resolvable department/subject. */
+export const UNCLASSIFIED = "ללא מחלקה";
+
+/** Max execution time (ms) for ticket statistics aggregations. */
+export const STATS_MAX_TIME_MS = 30_000;
+
+/** Default look-back window (days) for ticket statistics date ranges. */
+export const DEFAULT_LOOKBACK_DAYS = 30;

package/src/municipal/tickets/tickets.getters.ts

@@ -1,5 +1,4 @@
 import { CityName, getDb, ObjectId, Ticket } from "../index";
-import type { SubjectStatsItem } from "./tickets.types";
 import { Collection, Filter, ObjectId as MongoObjectId } from "mongodb";
 
 /**
@@ -68,145 +67,6 @@ export const deleteTicket = async (ticketId: string): Promise<boolean> => {
   return result.deletedCount > 0;
 };
 
-/**
- * Count tickets by city and date range (createdAt converted to given timezone).
- * Used as "open" tickets count when status is not available.
- */
-export async function getTicketsCountByCityAndDateRange(
-  cityName: string,
-  startStr: string,
-  endStr: string,
-  timezone: string,
-): Promise<number> {
-  const doc = await getTicketsCollection()
-    .aggregate<{ n: number }>([
-      { $match: { cityName, callSid: { $exists: true, $nin: [null, ""] } } },
-      {
-        $addFields: {
-          dateLocal: {
-            $dateToString: { format: "%Y-%m-%d", date: "$createdAt", timezone },
-          },
-        },
-      },
-      { $match: { dateLocal: { $gte: startStr, $lte: endStr } } },
-      { $count: "n" },
-    ])
-    .next();
-  return doc?.n ?? 0;
-}
-
-/**
- * Count of tickets by department (subject) from departmentsSubjects. Different departments (subject_id) are grouped, and sub-subjects are unified under their department.
- * Date filter: createdAt in [startStr, endStr] (in the given timezone). Fallback to "Unclassified" when no match is found in the lookup.
- */
-export async function getTicketsSubjectStats(
-  cityName: string,
-  startStr: string,
-  endStr: string,
-  timezone: string,
-): Promise<SubjectStatsItem[]> {
-  const coll = getTicketsCollection();
-  const rows = await coll
-    .aggregate<{ _id: string; subject: string; count: number }>([
-      { $match: { cityName } },
-      {
-        $addFields: {
-          dateLocal: {
-            $dateToString: { format: "%Y-%m-%d", date: "$createdAt", timezone },
-          },
-        },
-      },
-      { $match: { dateLocal: { $gte: startStr, $lte: endStr } } },
-      {
-        $addFields: {
-          effectiveSubjectId: {
-            $ifNull: [
-              "$externalCallFields.event_subject_id",
-              {
-                $ifNull: [
-                  "$externalCallFields.event_sub_subject_id",
-                  "$externalCallFields.event_sub_subject_id2",
-                ],
-              },
-            ],
-          },
-        },
-      },
-      {
-        $lookup: {
-          from: "departmentsSubjects",
-          let: { sid: "$effectiveSubjectId", c: cityName },
-          pipeline: [
-            {
-              $match: {
-                $expr: {
-                  $and: [
-                    { $eq: ["$cityName", "$$c"] },
-                    {
-                      $or: [
-                        { $eq: ["$subject_id", { $ifNull: ["$$sid", ""] }] },
-                        {
-                          $eq: ["$sub_subject_id", { $ifNull: ["$$sid", ""] }],
-                        },
-                        {
-                          $eq: ["$sub_subject_id2", { $ifNull: ["$$sid", ""] }],
-                        },
-                      ],
-                    },
-                  ],
-                },
-              },
-            },
-            { $limit: 1 },
-          ],
-          as: "subj",
-        },
-      },
-      {
-        $addFields: {
-          subject_id: {
-            $cond: {
-              if: { $gt: [{ $size: "$subj" }, 0] },
-              then: {
-                $ifNull: [{ $arrayElemAt: ["$subj.subject_id", 0] }, ""],
-              },
-              else: "",
-            },
-          },
-          subject: {
-            $cond: {
-              if: { $gt: [{ $size: "$subj" }, 0] },
-              then: {
-                $ifNull: [
-                  { $arrayElemAt: ["$subj.subjectName", 0] },
-                  "Unclassified",
-                ],
-              },
-              else: "Unclassified",
-            },
-          },
-        },
-      },
-      {
-        $group: {
-          _id: "$subject_id",
-          subject: { $first: "$subject" },
-          count: { $sum: 1 },
-        },
-      },
-      { $sort: { count: -1 } },
-    ])
-    .toArray();
-
-  const total = rows.reduce((s: number, r: any) => s + r.count, 0);
-  return rows.map((r: any) => ({
-    subject_name: r.subject,
-    subject_id: r._id,
-    count: r.count,
-    percentage: total > 0 ? Math.round((r.count / total) * 100) : 0,
-  }));
-}
-
 export const findTicketByQuery = async (query: Partial<Ticket>) => {
   return await getTicketsCollection().findOne(query);
 };

package/src/municipal/tickets/tickets.statistics.aggregation.ts

@@ -0,0 +1,113 @@
+import { CityName } from "../utils/types";
+import { STATS_MAX_TIME_MS, UNCLASSIFIED } from "./tickets.constants";
+import { getTicketsCollection } from "./tickets.getters";
+import type { TicketStatsDateRange } from "./tickets.types";
+
+const effectiveSubjectIdExpr = {
+  $ifNull: [
+    "$externalCallFields.event_subject_id",
+    {
+      $ifNull: [
+        "$externalCallFields.event_sub_subject_id",
+        "$externalCallFields.event_sub_subject_id2",
+      ],
+    },
+  ],
+};
+
+const departmentsSubjectsLookup = (cityName: CityName) => ({
+  $lookup: {
+    from: "departmentsSubjects",
+    let: { sid: "$effectiveSubjectId", c: cityName },
+    pipeline: [
+      {
+        $match: {
+          $expr: {
+            $and: [
+              { $eq: ["$cityName", "$$c"] },
+              {
+                $or: [
+                  { $eq: ["$subject_id", { $ifNull: ["$$sid", ""] }] },
+                  { $eq: ["$sub_subject_id", { $ifNull: ["$$sid", ""] }] },
+                  { $eq: ["$sub_subject_id2", { $ifNull: ["$$sid", ""] }] },
+                ],
+              },
+            ],
+          },
+        },
+      },
+      { $limit: 1 },
+    ],
+    as: "subj",
+  },
+});
+
+const subjectLabelExpr = {
+  $cond: {
+    if: { $gt: [{ $size: "$subj" }, 0] },
+    then: {
+      $ifNull: [{ $arrayElemAt: ["$subj.subjectName", 0] }, UNCLASSIFIED],
+    },
+    else: UNCLASSIFIED,
+  },
+};
+
+const ticketDateMatch = (dateRange: TicketStatsDateRange) => ({
+  createdAt: { $gte: dateRange.from, $lte: dateRange.to },
+});
+
+export const isDraftSubjectIdExpr = {
+  $or: [
+    { $eq: [{ $type: "$externalCallFields.event_subject_id" }, "missing"] },
+    { $eq: ["$externalCallFields.event_subject_id", null] },
+    {
+      $regexMatch: {
+        input: { $ifNull: ["$externalCallFields.event_subject_id", ""] },
+        regex: /^\s*$/,
+      },
+    },
+  ],
+};
+
+export const ticketsWithCallSidMatch = (
+  cityName: CityName,
+  dateRange: TicketStatsDateRange,
+) => ({
+  cityName,
+  callSid: { $exists: true, $nin: [null, ""] },
+  ...ticketDateMatch(dateRange),
+});
+
+export const ticketsByCallSidsMatch = (
+  cityName: CityName,
+  callSids: string[],
+  dateRange?: TicketStatsDateRange,
+) => ({
+  cityName,
+  callSid: { $in: callSids },
+  ...(dateRange ? ticketDateMatch(dateRange) : {}),
+});
+
+export const subjectGroupStages = (
+  cityName: CityName,
+  limit?: number,
+): Record<string, unknown>[] => {
+  const shouldLimit = typeof limit === "number" && limit > 0;
+  return [
+    { $addFields: { effectiveSubjectId: effectiveSubjectIdExpr } },
+    departmentsSubjectsLookup(cityName),
+    { $addFields: { subject: subjectLabelExpr } },
+    { $group: { _id: "$subject", count: { $sum: 1 } } },
+    { $sort: { count: -1 } },
+    ...(shouldLimit ? [{ $limit: limit }] : []),
+  ];
+};
+
+export const runTicketStatsAggregate = async <T>(
+  pipeline: Record<string, unknown>[],
+): Promise<T[]> => {
+  const rows = await getTicketsCollection()
+    .aggregate(pipeline, { maxTimeMS: STATS_MAX_TIME_MS })
+    .toArray();
+  return rows as T[];
+};

package/src/municipal/tickets/tickets.statistics.getters.ts

@@ -0,0 +1,93 @@
+import { CityName } from "../utils/types";
+import {
+  DAY_MS,
+  endOfCalendarDayInTz,
+  startOfCalendarDayInTz,
+} from "../../utils/date.utils";
+import { DEFAULT_LOOKBACK_DAYS } from "./tickets.constants";
+import {
+  isDraftSubjectIdExpr,
+  runTicketStatsAggregate,
+  subjectGroupStages,
+  ticketsByCallSidsMatch,
+  ticketsWithCallSidMatch,
+} from "./tickets.statistics.aggregation";
+import type {
+  SubjectItem,
+  TicketStatsDateRange,
+  TicketStatsDateScope,
+} from "./tickets.types";
+
+export const resolveTicketStatsDateRange = (
+  dateScope?: TicketStatsDateScope,
+): TicketStatsDateRange => {
+  const to = dateScope?.to ?? new Date();
+  const from =
+    dateScope?.from ?? new Date(to.getTime() - DEFAULT_LOOKBACK_DAYS * DAY_MS);
+  return { from, to };
+};
+
+export const ticketStatsDateScopeFromYmd = (
+  startStr: string,
+  endStr: string,
+  timezone: string,
+): TicketStatsDateScope => ({
+  from: startOfCalendarDayInTz(startStr, timezone),
+  to: endOfCalendarDayInTz(endStr, timezone),
+});
+
+export const findCallSidsWithTicketsByCity = async (
+  cityName: CityName,
+  dateScope?: TicketStatsDateScope,
+): Promise<string[]> => {
+  const rows = await runTicketStatsAggregate<{ _id: unknown }>([
+    { $match: ticketsWithCallSidMatch(cityName, resolveTicketStatsDateRange(dateScope)) },
+    { $group: { _id: "$callSid" } },
+  ]);
+  return rows.map((r) => String(r._id)).filter(Boolean);
+};
+
+export const findCallSidsWithDraftTicketsByCity = async (
+  cityName: CityName,
+  dateScope?: TicketStatsDateScope,
+): Promise<string[]> => {
+  const rows = await runTicketStatsAggregate<{ _id: unknown }>([
+    { $match: ticketsWithCallSidMatch(cityName, resolveTicketStatsDateRange(dateScope)) },
+    { $match: { $expr: isDraftSubjectIdExpr } },
+    { $group: { _id: "$callSid" } },
+  ]);
+  return rows.map((r) => String(r._id)).filter(Boolean);
+};
+
+export const findSubjectsByCityAndDateRange = async (
+  cityName: CityName,
+  dateScope?: TicketStatsDateScope,
+  limit?: number,
+): Promise<SubjectItem[]> => {
+  const rows = await runTicketStatsAggregate<{ _id: unknown; count: number }>([
+    { $match: ticketsWithCallSidMatch(cityName, resolveTicketStatsDateRange(dateScope)) },
+    ...subjectGroupStages(cityName, limit),
+  ]);
+  return rows.map((r) => ({ subject: String(r._id), count: r.count }));
+};
+
+export const findSubjectsByCallSids = async (
+  cityName: CityName,
+  callSids: string[],
+  limit?: number,
+  dateScope?: TicketStatsDateScope,
+): Promise<SubjectItem[]> => {
+  if (!callSids.length) return [];
+
+  const rows = await runTicketStatsAggregate<{ _id: unknown; count: number }>([
+    {
+      $match: ticketsByCallSidsMatch(
+        cityName,
+        callSids,
+        dateScope ? resolveTicketStatsDateRange(dateScope) : undefined,
+      ),
+    },
+    ...subjectGroupStages(cityName, limit),
+  ]);
+  return rows.map((r) => ({ subject: String(r._id), count: r.count }));
+};

package/src/municipal/tickets/tickets.types.ts

@@ -9,7 +9,6 @@ export type Ticket = {
   callSid?: string;
   cityName: CityName;
   externalCallFields: {
-    // Request fields
     first_name?: string;
     last_name?: string;
     event_description?: string;
@@ -18,10 +17,9 @@ export type Ticket = {
     event_subject_id?: string;
     event_sub_subject_id?: string;
     event_sub_subject_id2?: string | null;
-    // Response fields (only if external call was successful)
     call_number?: string;
-    status?: number; // Response status from Bina API (1 = success)
-    error?: string; // Error message from Bina API if any
+    status?: number;
+    error?: string;
     image1?: string;
     image2?: string;
     image3?: string;
@@ -34,10 +32,17 @@ export type Ticket = {
 
 export type TicketDoc = WithId<Ticket>;
 
-/** Call count by department (subject). */
-export type SubjectStatsItem = {
-  subject_name: string;
-  subject_id: string;
+export type SubjectItem = {
+  subject: string;
   count: number;
-  percentage: number;
+};
+
+export type TicketStatsDateScope = {
+  from?: Date;
+  to?: Date;
+};
+
+export type TicketStatsDateRange = {
+  from: Date;
+  to: Date;
 };