@trafficgroup/knex-rel 0.1.0 → 0.1.2

package/folder_cameraid_analysis.md

@@ -0,0 +1,167 @@
+# Folder Table CameraId Analysis
+
+## Issue Summary
+
+The `cameraId` column in the `folders` table is being saved as `null` despite being defined as optional in the schema and interface.
+
+## Database Schema Analysis
+
+### Current Schema (from migrations)
+
+#### Initial Folder Table Creation (20250717161310_migration.ts)
+
+```sql
+CREATE TABLE folders (
+  id SERIAL PRIMARY KEY,
+  name VARCHAR NOT NULL,
+  createdBy INTEGER NOT NULL REFERENCES user(id) ON DELETE CASCADE,
+  status ENUM('UPLOADING', 'COMPLETE') NOT NULL DEFAULT 'UPLOADING',
+  studyId INTEGER NOT NULL REFERENCES study(id) ON DELETE CASCADE,
+  created_at TIMESTAMP,
+  updated_at TIMESTAMP
+);
+```
+
+#### Camera Integration (20250911000000_migration.ts)
+
+```sql
+-- Added cameraId column to folders table
+ALTER TABLE folders ADD COLUMN cameraId INTEGER NULLABLE
+  REFERENCES cameras(id) ON DELETE SET NULL;
+CREATE INDEX ON folders (cameraId);
+```
+
+### Current Interface Definition
+
+```typescript
+export interface IFolder {
+  id: number;
+  uuid: string;
+  name: string;
+  createdBy: number; // user.id
+  status: "UPLOADING" | "COMPLETE";
+  studyId: number; // study.id
+  cameraId?: number; // camera.id - OPTIONAL field
+  created_at: string;
+  updated_at: string;
+  study?: IStudy;
+  camera?: ICamera;
+}
+```
+
+## DAO Implementation Analysis
+
+### Current FolderDAO Issues
+
+1. **No Camera Relationship Joins**: The DAO queries join with `study` table but don't include `camera` joins
+2. **Missing Camera Data Population**: Queries don't populate the optional `camera` field
+3. **Basic CRUD Operations**: The `create()` method simply inserts the provided data without validation
+
+### Key Methods Analysis
+
+#### Create Method
+
+```typescript
+async create(item: IFolder): Promise<IFolder> {
+  const [createdFolder] = await this._knex("folders")
+    .insert(item)
+    .returning("*");
+  return createdFolder;
+}
+```
+
+- **Issue**: This will insert whatever `cameraId` is provided in the `item` parameter
+- **Behavior**: If `cameraId` is undefined or not provided, it will be stored as `NULL` (which is valid)
+
+#### Query Methods (getById, getByUuid, getAll)
+
+```typescript
+// Current pattern - only joins with study
+const folder = await this._knex("folders as f")
+  .innerJoin("study as s", "f.studyId", "s.id")
+  .select("f.*", this._knex.raw("to_jsonb(s.*) as study"))
+  .where("f.id", id)
+  .first();
+```
+
+- **Missing**: No LEFT JOIN with cameras table to populate camera data
+- **Impact**: Even if `cameraId` has a valid value, the `camera` object won't be populated
+
+## Root Cause Analysis
+
+### Why CameraId is Null
+
+1. **Optional Field Behavior**: Since `cameraId` is optional (`cameraId?: number`), if not explicitly provided in create requests, it defaults to `null`
+
+2. **No Default Value**: Unlike other fields, there's no database-level default value for `cameraId`
+
+3. **Application Logic Gap**: No validation or business logic to ensure `cameraId` is populated when appropriate
+
+### Database Schema is Correct
+
+The database schema properly supports `cameraId`:
+
+- ✅ Column exists and is nullable
+- ✅ Foreign key constraint to `cameras(id)`
+- ✅ Proper ON DELETE SET NULL behavior
+- ✅ Index created for performance
+
+## Recommended Solutions
+
+### 1. Immediate Fix: Verify Input Data
+
+Check what data is being passed to the `create()` method:
+
+- Log the `item` parameter in FolderDAO.create()
+- Verify if `cameraId` is being provided in the request
+
+### 2. Enhanced DAO Queries
+
+Update FolderDAO to include camera relationships:
+
+```typescript
+// Enhanced getById with camera join
+async getById(id: number): Promise<IFolder | null> {
+  const folder = await this._knex("folders as f")
+    .innerJoin("study as s", "f.studyId", "s.id")
+    .leftJoin("cameras as c", "f.cameraId", "c.id")
+    .select(
+      "f.*",
+      this._knex.raw("to_jsonb(s.*) as study"),
+      this._knex.raw("to_jsonb(c.*) as camera")
+    )
+    .where("f.id", id)
+    .first();
+  return folder || null;
+}
+```
+
+### 3. API Layer Investigation
+
+Check the API endpoints that create folders:
+
+- Verify if `cameraId` is being extracted from request body
+- Ensure DTO/validation layer passes `cameraId` through
+- Check if frontend is sending `cameraId` in requests
+
+### 4. Business Logic Validation
+
+Consider if folders should always have a camera:
+
+- If required: Add validation in create method
+- If optional: Current behavior is correct (null is valid)
+
+## Implementation Priority
+
+1. **High**: Investigate API request data flow
+2. **High**: Add camera relationship to DAO queries
+3. **Medium**: Add logging to track cameraId values
+4. **Low**: Consider business logic changes
+
+## Files to Examine Next
+
+1. `api-rel/src/controllers/folder/folder.controller.ts` - Controller logic
+2. `api-rel/src/routes/folder/` - Route definitions and DTOs
+3. Frontend folder creation forms - Check if cameraId is being sent
+
+The database schema and DAO patterns are correctly implemented. The issue likely lies in the data flow from the API/frontend layers not providing `cameraId` values during folder creation.

package/migrations/20250911000000_migration.ts

@@ -0,0 +1,61 @@
+import type { Knex } from "knex";
+
+export async function up(knex: Knex): Promise<void> {
+  // Create cameras table
+  await knex.schema.createTable("cameras", (table) => {
+    table.increments("id").primary();
+    table
+      .uuid("uuid")
+      .defaultTo(knex.raw("uuid_generate_v4()"))
+      .notNullable()
+      .unique();
+    table.string("name", 100).notNullable();
+    table.decimal("longitude", 10, 7).notNullable();
+    table.decimal("latitude", 10, 7).notNullable();
+    table.timestamps(true, true);
+
+    table.index(["uuid"]);
+    table.index(["longitude", "latitude"]);
+  });
+
+  // Add cameraId to videos table
+  await knex.schema.alterTable("video", (table) => {
+    table
+      .integer("cameraId")
+      .nullable()
+      .references("id")
+      .inTable("cameras")
+      .onDelete("SET NULL");
+    table.index(["cameraId"]);
+  });
+
+  // Add cameraId to folders table
+  await knex.schema.alterTable("folders", (table) => {
+    table
+      .integer("cameraId")
+      .nullable()
+      .references("id")
+      .inTable("cameras")
+      .onDelete("SET NULL");
+    table.index(["cameraId"]);
+  });
+}
+
+export async function down(knex: Knex): Promise<void> {
+  // Remove cameraId from folders table
+  await knex.schema.alterTable("folders", (table) => {
+    table.dropIndex(["cameraId"]);
+    table.dropForeign(["cameraId"]);
+    table.dropColumn("cameraId");
+  });
+
+  // Remove cameraId from videos table
+  await knex.schema.alterTable("video", (table) => {
+    table.dropIndex(["cameraId"]);
+    table.dropForeign(["cameraId"]);
+    table.dropColumn("cameraId");
+  });
+
+  // Drop cameras table
+  await knex.schema.dropTable("cameras");
+}

package/migrations/20250917144153_migration.ts

@@ -0,0 +1,37 @@
+import type { Knex } from "knex";
+
+export async function up(knex: Knex): Promise<void> {
+  await knex.schema.alterTable("video", (table) => {
+    // Add annotationSourceId column with foreign key to video table
+    table
+      .integer("annotationSourceId")
+      .nullable()
+      .references("id")
+      .inTable("video")
+      .onDelete("SET NULL");
+
+    // Add index for performance when querying by annotation source
+    table.index(["annotationSourceId"], "idx_video_annotation_source_id");
+
+    // Add composite index for efficient template lookups
+    table.index(
+      ["folderId", "videoType", "status"],
+      "idx_video_template_lookup",
+    );
+  });
+}
+
+export async function down(knex: Knex): Promise<void> {
+  await knex.schema.alterTable("video", (table) => {
+    // Drop indexes first
+    table.dropIndex(
+      ["folderId", "videoType", "status"],
+      "idx_video_template_lookup",
+    );
+    table.dropIndex(["annotationSourceId"], "idx_video_annotation_source_id");
+
+    // Drop foreign key and column
+    table.dropForeign(["annotationSourceId"]);
+    table.dropColumn("annotationSourceId");
+  });
+}

package/migrations/20250924000000_camera_name_search_index.ts

@@ -0,0 +1,22 @@
+import type { Knex } from "knex";
+
+export async function up(knex: Knex): Promise<void> {
+  await knex.schema.alterTable("cameras", (table) => {
+    // Add index for case-insensitive name searches
+    table.index(knex.raw("LOWER(name)"), "idx_cameras_name_lower");
+  });
+
+  // Add GIN index for full-text search if needed for fuzzy matching
+  await knex.raw(`
+    CREATE INDEX idx_cameras_name_gin
+    ON cameras
+    USING GIN (to_tsvector('english', name))
+  `);
+}
+
+export async function down(knex: Knex): Promise<void> {
+  await knex.raw("DROP INDEX IF EXISTS idx_cameras_name_gin");
+  await knex.schema.alterTable("cameras", (table) => {
+    table.dropIndex(knex.raw("LOWER(name)"), "idx_cameras_name_lower");
+  });
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@trafficgroup/knex-rel",
-  "version": "0.1.0",
+  "version": "0.1.2",
   "description": "Knex Module",
   "main": "dist/index.js",
   "scripts": {

package/plan.md

@@ -0,0 +1,288 @@
+# Database Optimization Plan for Camera Bulk Operations
+
+## Current Schema Analysis
+
+### Existing Tables & Indexes
+
+**cameras table:**
+
+- Primary key: `id` (auto-increment)
+- Unique index: `uuid`
+- Composite index: `[longitude, latitude]` for geospatial queries
+
+**video table:**
+
+- Primary key: `id`
+- Foreign key: `cameraId` → `cameras.id` (nullable, with index)
+- Foreign key: `folderId` → `folders.id` (with implicit index)
+- Unique index: `uuid`
+- Composite indexes:
+  - `[annotationSourceId]`
+  - `[folderId, videoType, status]` (template lookup)
+
+**folders table:**
+
+- Primary key: `id`
+- Foreign key: `cameraId` → `cameras.id` (nullable, with index)
+- Foreign key: `studyId` → `study.id` (with implicit index)
+- Unique index: `uuid`
+
+### Performance Analysis for Required Operations
+
+#### 1. ✅ Bulk update videos when folder cameraId changes
+
+**Query:** `UPDATE video SET cameraId = ? WHERE folderId = ?`
+
+- **Current Performance:** GOOD
+- **Existing Index:** `[folderId]` (implicit from FK constraint)
+- **Status:** No optimization needed
+
+#### 2. ❌ Search cameras by name with pagination
+
+**Query:** `SELECT * FROM cameras WHERE name ILIKE '%search%' ORDER BY name LIMIT ? OFFSET ?`
+
+- **Current Performance:** POOR - Full table scan
+- **Missing Index:** Text search index on `name` column
+- **Impact:** Critical performance issue for camera search
+
+#### 3. ✅ Find videos by specific camera
+
+**Query:** `SELECT * FROM video WHERE cameraId = ?`
+
+- **Current Performance:** GOOD
+- **Existing Index:** `[cameraId]` from migration `20250911000000_migration.ts`
+- **Status:** No optimization needed
+
+#### 4. ⚠️ Bulk assign camera to multiple videos
+
+**Query:** `UPDATE video SET cameraId = ? WHERE id IN (...)`
+
+- **Current Performance:** ACCEPTABLE but can be optimized
+- **Existing Index:** Primary key on `id`
+- **Optimization Opportunity:** Batch processing with transaction optimization
+
+## Required Database Optimizations
+
+### New Migrations Required
+
+#### Migration: Add camera name search index
+
+**File:** `20250924000000_camera_name_search_index.ts`
+**Purpose:** Optimize camera search by name functionality
+
+```typescript
+export async function up(knex: Knex): Promise<void> {
+  await knex.schema.alterTable("cameras", (table) => {
+    // Add index for case-insensitive name searches
+    table.index(knex.raw("LOWER(name)"), "idx_cameras_name_lower");
+  });
+
+  // Add GIN index for full-text search if needed for fuzzy matching
+  await knex.raw(`
+    CREATE INDEX idx_cameras_name_gin
+    ON cameras
+    USING GIN (to_tsvector('english', name))
+  `);
+}
+
+export async function down(knex: Knex): Promise<void> {
+  await knex.raw("DROP INDEX IF EXISTS idx_cameras_name_gin");
+  await knex.schema.alterTable("cameras", (table) => {
+    table.dropIndex(knex.raw("LOWER(name)"), "idx_cameras_name_lower");
+  });
+}
+```
+
+### DAO Method Optimizations
+
+#### CameraDAO Enhancements Required
+
+**New Methods Needed:**
+
+```typescript
+// Optimized paginated search by name
+async searchByName(
+  searchTerm: string,
+  page: number,
+  limit: number
+): Promise<IDataPaginator<ICamera>> {
+  const offset = (page - 1) * limit;
+  const searchPattern = `%${searchTerm.toLowerCase()}%`;
+
+  const query = this._knex("cameras")
+    .whereRaw("LOWER(name) LIKE ?", [searchPattern])
+    .orderBy("name");
+
+  const [countResult] = await query.clone().clearSelect().count("* as count");
+  const totalCount = +countResult.count;
+  const cameras = await query.clone().limit(limit).offset(offset);
+
+  return {
+    success: true,
+    data: cameras,
+    page,
+    limit,
+    count: cameras.length,
+    totalCount,
+    totalPages: Math.ceil(totalCount / limit),
+  };
+}
+
+// Get all videos associated with a camera (with pagination)
+async getVideosByCamera(
+  cameraId: number,
+  page: number,
+  limit: number
+): Promise<IDataPaginator<IVideo>> {
+  const offset = (page - 1) * limit;
+
+  const query = this._knex("video as v")
+    .innerJoin("folders as f", "v.folderId", "f.id")
+    .select("v.*", this._knex.raw("to_jsonb(f.*) as folder"))
+    .where("v.cameraId", cameraId)
+    .orderBy("v.created_at", "desc");
+
+  const [countResult] = await query.clone().clearSelect().count("* as count");
+  const totalCount = +countResult.count;
+  const videos = await query.clone().limit(limit).offset(offset);
+
+  return {
+    success: true,
+    data: videos,
+    page,
+    limit,
+    count: videos.length,
+    totalCount,
+    totalPages: Math.ceil(totalCount / limit),
+  };
+}
+```
+
+#### VideoDAO Enhancements Required
+
+**New Methods Needed:**
+
+```typescript
+// Bulk update videos by folder (folder camera cascade)
+async bulkUpdateCameraByFolder(
+  folderId: number,
+  cameraId: number | null
+): Promise<number> {
+  const result = await this._knex("video")
+    .where("folderId", folderId)
+    .update({
+      cameraId: cameraId,
+      updated_at: this._knex.fn.now()
+    });
+  return result;
+}
+
+// Bulk assign camera to multiple videos
+async bulkUpdateCamera(
+  videoIds: number[],
+  cameraId: number | null
+): Promise<number> {
+  if (videoIds.length === 0) return 0;
+
+  const result = await this._knex("video")
+    .whereIn("id", videoIds)
+    .update({
+      cameraId: cameraId,
+      updated_at: this._knex.fn.now()
+    });
+  return result;
+}
+
+// Get videos by multiple folder IDs (for folder camera propagation)
+async getVideosByFolderIds(folderIds: number[]): Promise<IVideo[]> {
+  if (folderIds.length === 0) return [];
+
+  return await this._knex("video")
+    .whereIn("folderId", folderIds)
+    .select("id", "uuid", "name", "folderId", "cameraId");
+}
+```
+
+## Query Performance Strategies
+
+### 1. Camera Search Optimization
+
+- **Index Strategy:** Case-insensitive BTREE index on `LOWER(name)`
+- **Query Pattern:** `WHERE LOWER(name) LIKE LOWER(?)`
+- **Full-text Option:** GIN index with `to_tsvector()` for fuzzy search
+- **Expected Performance:** O(log n) lookup instead of O(n) scan
+
+### 2. Bulk Operation Optimization
+
+- **Transaction Wrapping:** All bulk operations in single transactions
+- **Batch Size Limits:** Process in chunks of 1000 records max
+- **Index Usage:** Leverage existing primary key and foreign key indexes
+
+### 3. Folder Camera Cascade Optimization
+
+- **Single Query Strategy:** Use `WHERE folderId = ?` leveraging existing index
+- **Atomic Updates:** Single UPDATE statement rather than individual updates
+- **Expected Performance:** O(log n) folder lookup + O(k) video updates
+
+## Implementation Order
+
+1. **Migration First:** Deploy camera name search index
+2. **DAO Methods:** Add optimized search and bulk operation methods
+3. **Transaction Optimization:** Implement proper transaction handling
+4. **Testing:** Validate performance improvements
+
+## Risk Assessment
+
+### Migration Safety: LOW RISK
+
+- Adding indexes is non-blocking operation
+- No data changes, only performance improvements
+- Easy rollback with down migration
+
+### Performance Impact: HIGH BENEFIT
+
+- Camera search: 100x-1000x improvement (O(n) → O(log n))
+- Bulk operations: Proper indexing already exists
+- Folder cascade: Leverages existing folderId index
+
+### Pattern Compliance: 100% COMPLIANT
+
+- Follows existing DAO patterns exactly
+- Maintains IBaseDAO interface requirements
+- Uses established naming conventions
+- Preserves UUID external API pattern
+
+## Validation Strategy
+
+### Performance Testing Queries
+
+```sql
+-- Test camera search performance
+EXPLAIN ANALYZE SELECT * FROM cameras
+WHERE LOWER(name) LIKE '%traffic%'
+ORDER BY name LIMIT 20;
+
+-- Test video-by-camera lookup
+EXPLAIN ANALYZE SELECT * FROM video
+WHERE cameraId = 1
+ORDER BY created_at DESC LIMIT 20;
+
+-- Test folder cascade update
+EXPLAIN ANALYZE UPDATE video
+SET cameraId = 1 WHERE folderId = 5;
+```
+
+### Success Metrics
+
+- Camera search < 50ms for 10k+ cameras
+- Video-by-camera < 100ms for 1k+ videos
+- Folder cascade < 200ms for 100+ videos per folder
+- Bulk video update < 500ms for 1000+ videos
+
+## Conclusion
+
+**Current Status:** Schema is well-designed with most needed indexes already present. Only critical gap is camera name search optimization.
+
+**Priority:** HIGH for camera name search index, MEDIUM for DAO method enhancements.
+
+**Complexity:** LOW - Single migration required, standard DAO patterns.

package/src/dao/camera/camera.dao.ts

@@ -0,0 +1,79 @@
+import { Knex } from "knex";
+import { IBaseDAO, IDataPaginator } from "../../d.types";
+import { ICamera } from "../../interfaces/camera/camera.interfaces";
+import KnexManager from "../../KnexConnection";
+
+export class CameraDAO implements IBaseDAO<ICamera> {
+  private _knex: Knex<any, unknown[]> = KnexManager.getConnection();
+
+  async create(item: ICamera): Promise<ICamera> {
+    const [createdCamera] = await this._knex("cameras")
+      .insert(item)
+      .returning("*");
+    return createdCamera;
+  }
+
+  async getById(id: number): Promise<ICamera | null> {
+    const camera = await this._knex("cameras").where({ id }).first();
+    return camera || null;
+  }
+
+  async getByUuid(uuid: string): Promise<ICamera | null> {
+    const camera = await this._knex("cameras").where({ uuid }).first();
+    return camera || null;
+  }
+
+  async update(id: number, item: Partial<ICamera>): Promise<ICamera | null> {
+    const [updatedCamera] = await this._knex("cameras")
+      .where({ id })
+      .update(item)
+      .returning("*");
+    return updatedCamera || null;
+  }
+
+  async delete(id: number): Promise<boolean> {
+    const result = await this._knex("cameras").where({ id }).del();
+    return result > 0;
+  }
+
+  async getAll(page: number, limit: number): Promise<IDataPaginator<ICamera>> {
+    const offset = (page - 1) * limit;
+
+    const [countResult] = await this._knex("cameras").count("* as count");
+    const totalCount = +countResult.count;
+    const cameras = await this._knex("cameras").limit(limit).offset(offset);
+
+    return {
+      success: true,
+      data: cameras,
+      page,
+      limit,
+      count: cameras.length,
+      totalCount,
+      totalPages: Math.ceil(totalCount / limit),
+    };
+  }
+
+  async getByName(name: string): Promise<ICamera | null> {
+    const camera = await this._knex("cameras").where({ name }).first();
+    return camera || null;
+  }
+
+  async getCamerasNearCoordinates(
+    longitude: number,
+    latitude: number,
+    radiusKm: number = 1,
+  ): Promise<ICamera[]> {
+    // Using ST_DWithin for geographic distance calculation
+    // This is a PostgreSQL-specific query for geospatial operations
+    const cameras = await this._knex("cameras").whereRaw(
+      `ST_DWithin(
+                    ST_MakePoint(longitude, latitude)::geography,
+                    ST_MakePoint(?, ?)::geography,
+                    ?
+                )`,
+      [longitude, latitude, radiusKm * 1000], // Convert km to meters
+    );
+    return cameras;
+  }
+}