@trafficgroup/knex-rel 0.0.28 → 0.0.29

package/plan.md

@@ -0,0 +1,304 @@
+# Database Implementation Plan: Video Minute Results Storage
+
+## Schema Changes
+
+### New Table: video_minute_results
+
+```sql
+CREATE TABLE video_minute_results (
+  id BIGSERIAL PRIMARY KEY,
+  uuid UUID NOT NULL DEFAULT gen_random_uuid(),
+  video_id INTEGER NOT NULL REFERENCES video(id) ON DELETE CASCADE,
+  minute_number INTEGER NOT NULL,
+  results JSONB NOT NULL,
+  created_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP,
+  updated_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP,
+  CONSTRAINT unique_video_minute UNIQUE (video_id, minute_number),
+  CONSTRAINT valid_minute_number CHECK (minute_number >= 0)
+);
+```
+
+**Indexes:**
+
+- `idx_video_minute_results_video_id` on `video_id` (frequent JOINs with video table)
+- `idx_video_minute_results_uuid` on `uuid` (external API lookups)
+- `idx_video_minute_results_minute_number` on `minute_number` (range queries)
+- `idx_video_minute_results_video_minute` on `(video_id, minute_number)` (composite lookups)
+
+### Modifications to Existing Tables
+
+```sql
+-- Add duration tracking to video table
+ALTER TABLE video ADD COLUMN duration_seconds INTEGER;
+```
+
+## Migrations
+
+### 20250823HHMMSS_create_video_minute_results.ts
+
+```typescript
+export async function up(knex: Knex): Promise<void> {
+  // Create video_minute_results table
+  await knex.schema.createTable("video_minute_results", (table) => {
+    table.bigIncrements("id").primary();
+    table
+      .uuid("uuid")
+      .defaultTo(knex.raw("gen_random_uuid()"))
+      .notNullable()
+      .unique();
+    table
+      .integer("video_id")
+      .unsigned()
+      .notNullable()
+      .references("id")
+      .inTable("video")
+      .onDelete("CASCADE");
+    table.integer("minute_number").notNullable();
+    table.jsonb("results").notNullable();
+    table.timestamps(true, true);
+
+    // Constraints
+    table.unique(["video_id", "minute_number"], {
+      indexName: "unique_video_minute",
+    });
+    table.check("minute_number >= 0", [], "valid_minute_number");
+
+    // Indexes
+    table.index("video_id", "idx_video_minute_results_video_id");
+    table.index("uuid", "idx_video_minute_results_uuid");
+    table.index("minute_number", "idx_video_minute_results_minute_number");
+    table.index(
+      ["video_id", "minute_number"],
+      "idx_video_minute_results_video_minute",
+    );
+  });
+
+  // Add duration_seconds to video table
+  await knex.schema.alterTable("video", (table) => {
+    table.integer("duration_seconds").nullable();
+  });
+}
+
+export async function down(knex: Knex): Promise<void> {
+  await knex.schema.dropTableIfExists("video_minute_results");
+  await knex.schema.alterTable("video", (table) => {
+    table.dropColumn("duration_seconds");
+  });
+}
+```
+
+**Safety Level:** Medium - Creates new table and adds nullable column to existing table
+
+## DAOs
+
+### Update: VideoMinuteResultDAO
+
+**File:** `src/dao/VideoMinuteResultDAO.ts` (replace existing mock implementation)
+
+**Methods:**
+
+```typescript
+export class VideoMinuteResultDAO implements IBaseDAO<IVideoMinuteResult> {
+  private knex = KnexManager.getConnection();
+  private tableName = "video_minute_results";
+
+  // Standard CRUD operations
+  async create(data: IVideoMinuteResultInput): Promise<IVideoMinuteResult>;
+  async getById(id: number): Promise<IVideoMinuteResult | null>;
+  async getByUuid(uuid: string): Promise<IVideoMinuteResult | null>;
+  async getAll(
+    page: number,
+    limit: number,
+  ): Promise<IDataPaginator<IVideoMinuteResult>>;
+  async update(
+    id: number,
+    data: Partial<IVideoMinuteResult>,
+  ): Promise<IVideoMinuteResult | null>;
+  async delete(id: number): Promise<boolean>;
+
+  // Specialized batch operations (eliminate N+1 queries)
+  async createBatch(
+    videoId: number,
+    minuteResults: IVideoMinuteResultInput[],
+  ): Promise<IVideoMinuteResult[]>;
+  async getMinuteResultsForVideo(
+    videoId: number,
+    startMinute?: number,
+    endMinute?: number,
+    page?: number,
+    limit?: number,
+  ): Promise<IDataPaginator<IVideoMinuteResult>>;
+  async getMinuteResultsByVideoUuid(
+    videoUuid: string,
+    startMinute?: number,
+    endMinute?: number,
+    page?: number,
+    limit?: number,
+  ): Promise<IDataPaginator<IVideoMinuteResult>>;
+  async deleteByVideoId(videoId: number): Promise<boolean>;
+  async getVideoMinuteRange(
+    videoId: number,
+  ): Promise<{ minMinute: number; maxMinute: number } | null>;
+}
+```
+
+**Performance Optimizations:**
+
+- Batch inserts using `knex.batchInsert()` for 5-minute batches
+- Single query with JOINs for video+minute data retrieval
+- Range queries with BETWEEN for minute filtering
+- Pagination with proper OFFSET/LIMIT
+
+### Update: VideoDAO
+
+**File:** `src/dao/video/video.dao.ts` (add duration_seconds support)
+
+**New Methods:**
+
+```typescript
+async updateDuration(id: number, durationSeconds: number): Promise<IVideo | null>
+async getVideosWithoutMinuteData(page: number, limit: number): Promise<IDataPaginator<IVideo>>
+```
+
+## Interfaces
+
+### Update: IVideoMinuteResult.ts
+
+```typescript
+export interface IVideoMinuteResult extends IBaseEntity {
+  id: number;
+  uuid: string;
+  videoId: number;
+  minuteNumber: number; // Renamed from 'minute' for clarity
+  results: Record<string, any>;
+  createdAt: string;
+  updatedAt: string;
+}
+
+export interface IVideoMinuteResultInput {
+  videoId: number;
+  minuteNumber: number;
+  results: Record<string, any>;
+}
+
+export interface IVideoMinuteBatch {
+  videoId: number;
+  startMinute: number;
+  endMinute: number;
+  minuteResults: IVideoMinuteResultInput[];
+}
+```
+
+### Update: IVideo.ts
+
+```typescript
+export interface IVideo {
+  // ... existing fields
+  durationSeconds?: number; // Add optional duration field
+}
+```
+
+### Export Updates
+
+Update `src/index.ts` to include new interfaces and updated DAO.
+
+## Implementation Order
+
+1. **Interfaces** → Update IVideoMinuteResult and IVideo interfaces
+2. **Migration** → Create 20250823HHMMSS_create_video_minute_results.ts
+3. **DAO Implementation** → Replace VideoMinuteResultDAO mock methods with real database operations
+4. **DAO Enhancement** → Add duration support to VideoDAO
+5. **Exports** → Update index.ts exports
+6. **Build & Test** → Compile TypeScript and run tests
+
+## Query Performance Strategy
+
+### Batch Insert Optimization
+
+```typescript
+async createBatch(videoId: number, minuteResults: IVideoMinuteResultInput[]): Promise<IVideoMinuteResult[]> {
+  const batchData = minuteResults.map(result => ({
+    ...result,
+    videoId,
+    uuid: knex.raw('gen_random_uuid()'),
+    created_at: knex.fn.now(),
+    updated_at: knex.fn.now()
+  }));
+
+  return await this.knex.batchInsert(this.tableName, batchData, 50).returning('*');
+}
+```
+
+### Optimized Minute Range Query
+
+```typescript
+async getMinuteResultsForVideo(
+  videoId: number,
+  startMinute: number = 0,
+  endMinute?: number,
+  page: number = 1,
+  limit: number = 100
+): Promise<IDataPaginator<IVideoMinuteResult>> {
+  const query = this.knex(this.tableName + ' as vmr')
+    .innerJoin('video as v', 'vmr.video_id', 'v.id')
+    .select('vmr.*', 'v.name as video_name', 'v.uuid as video_uuid')
+    .where('vmr.video_id', videoId)
+    .where('vmr.minute_number', '>=', startMinute);
+
+  if (endMinute !== undefined) {
+    query.where('vmr.minute_number', '<=', endMinute);
+  }
+
+  // Standard pagination logic with count optimization
+  const [{ count }] = await query.clone().clearSelect().count('* as count');
+  const offset = (page - 1) * limit;
+  const data = await query.orderBy('vmr.minute_number', 'asc').limit(limit).offset(offset);
+
+  return {
+    success: true,
+    data,
+    page,
+    limit,
+    count: data.length,
+    totalCount: parseInt(count as string),
+    totalPages: Math.ceil(parseInt(count as string) / limit)
+  };
+}
+```
+
+## Risks & Validation
+
+### Migration Safety
+
+- **Risk:** Table creation failure in production
+- **Mitigation:** Test migration on staging with production data volume
+- **Rollback:** `down()` function drops table and column safely
+
+### Pattern Compliance
+
+- **UUID-only External Communication:** ✅ All external API methods use UUID lookups
+- **Foreign Key Integrity:** ✅ CASCADE delete ensures orphaned records cleanup
+- **Naming Conventions:** ✅ Follows snake_case for DB, camelCase for TypeScript
+- **DAO Methods:** ✅ Standard CRUD + specialized batch operations
+- **Performance:** ✅ Proper indexing for all query patterns
+
+### Data Integrity Validation
+
+```typescript
+async validateMinuteDataIntegrity(videoId: number): Promise<boolean> {
+  const video = await this.knex('video').where('id', videoId).first();
+  const minuteResults = await this.knex(this.tableName).where('video_id', videoId);
+
+  // Aggregate minute results and compare with video.results
+  return this.compareAggregatedResults(video.results, minuteResults);
+}
+```
+
+### Performance Validation
+
+- Batch inserts handle 5-minute chunks efficiently (50-item batches)
+- Indexes support all query patterns: video_id lookups, UUID lookups, minute ranges
+- JOIN operations optimized for N+1 query elimination
+- Pagination implemented with proper count optimization
+
+This plan ensures 100% pattern compliance while optimizing for the specific use case of minute-by-minute video detection storage with efficient batch processing capabilities.

package/src/d.types.ts

@@ -1,18 +1,22 @@
-export interface IDataPaginator<T> {
-    success: boolean;
-    data: T[];
-    page: number;
-    limit: number;
-    count: number;
-    totalCount: number;
-    totalPages: number;
-}
-
-export interface IBaseDAO<T> {
-    create(item: T): Promise<T>;
-    getById(id: number): Promise<T | null>;
-    getByUuid(uuid: string): Promise<T | null>;
-    getAll(page: number, limit: number, entityId?: any | null): Promise<IDataPaginator<T>>;
-    update(id: number, item: T): Promise<T | null>;
-    delete(id: number): Promise<boolean>;
-}
+export interface IDataPaginator<T> {
+  success: boolean;
+  data: T[];
+  page: number;
+  limit: number;
+  count: number;
+  totalCount: number;
+  totalPages: number;
+}
+
+export interface IBaseDAO<T> {
+  create(item: T): Promise<T>;
+  getById(id: number): Promise<T | null>;
+  getByUuid(uuid: string): Promise<T | null>;
+  getAll(
+    page: number,
+    limit: number,
+    entityId?: any | null,
+  ): Promise<IDataPaginator<T>>;
+  update(id: number, item: T): Promise<T | null>;
+  delete(id: number): Promise<boolean>;
+}

package/src/dao/VideoMinuteResultDAO.ts

@@ -0,0 +1,273 @@
+import { IBaseDAO, IDataPaginator } from "../d.types";
+import {
+  IVideoMinuteResult,
+  IVideoMinuteResultInput,
+  IVideoMinuteBatch,
+} from "../entities/VideoMinuteResult";
+import KnexManager from "../KnexConnection";
+
+export class VideoMinuteResultDAO implements IBaseDAO<IVideoMinuteResult> {
+  private knex = KnexManager.getConnection();
+  private tableName = "video_minute_results";
+
+  async getAll(
+    page: number = 1,
+    limit: number = 10,
+  ): Promise<IDataPaginator<IVideoMinuteResult>> {
+    const offset = (page - 1) * limit;
+
+    const [countResult] = await this.knex(this.tableName).count("* as count");
+    const totalCount = +countResult.count;
+
+    const data = await this.knex(this.tableName)
+      .select("*")
+      .orderBy("created_at", "desc")
+      .limit(limit)
+      .offset(offset);
+
+    return {
+      success: true,
+      data: this.mapDbRowsToEntities(data),
+      page,
+      limit,
+      count: data.length,
+      totalCount,
+      totalPages: Math.ceil(totalCount / limit),
+    };
+  }
+
+  async getById(id: number): Promise<IVideoMinuteResult | null> {
+    const result = await this.knex(this.tableName).where("id", id).first();
+
+    return result ? this.mapDbRowToEntity(result) : null;
+  }
+
+  async getByUuid(uuid: string): Promise<IVideoMinuteResult | null> {
+    const result = await this.knex(this.tableName).where("uuid", uuid).first();
+
+    return result ? this.mapDbRowToEntity(result) : null;
+  }
+
+  async create(data: IVideoMinuteResultInput): Promise<IVideoMinuteResult> {
+    const dbData = this.mapEntityToDbRow(data);
+    const [result] = await this.knex(this.tableName)
+      .insert(dbData)
+      .returning("*");
+
+    return this.mapDbRowToEntity(result);
+  }
+
+  async update(
+    id: number,
+    data: IVideoMinuteResult,
+  ): Promise<IVideoMinuteResult | null> {
+    const dbData = this.mapEntityToDbRow(data);
+    const [result] = await this.knex(this.tableName)
+      .where("id", id)
+      .update({ ...dbData, updated_at: this.knex.fn.now() })
+      .returning("*");
+
+    return result ? this.mapDbRowToEntity(result) : null;
+  }
+
+  async delete(id: number): Promise<boolean> {
+    const result = await this.knex(this.tableName).where("id", id).del();
+
+    return result > 0;
+  }
+
+  /**
+   * Get minute results for a specific video with optional time range filtering
+   */
+  async getMinuteResultsForVideo(
+    videoId: number,
+    startMinute?: number,
+    endMinute?: number,
+    page: number = 1,
+    limit: number = 100,
+  ): Promise<IDataPaginator<IVideoMinuteResult>> {
+    const offset = (page - 1) * limit;
+
+    const query = this.knex(this.tableName + " as vmr")
+      .innerJoin("video as v", "vmr.video_id", "v.id")
+      .select("vmr.*", "v.name as video_name", "v.uuid as video_uuid")
+      .where("vmr.video_id", videoId);
+
+    if (startMinute !== undefined) {
+      query.where("vmr.minute_number", ">=", startMinute);
+    }
+
+    if (endMinute !== undefined) {
+      query.where("vmr.minute_number", "<=", endMinute);
+    }
+
+    // Standard pagination logic with count optimization
+    const [{ count }] = await query.clone().clearSelect().count("* as count");
+    const data = await query
+      .orderBy("vmr.minute_number", "asc")
+      .limit(limit)
+      .offset(offset);
+
+    return {
+      success: true,
+      data: this.mapDbRowsToEntities(data),
+      page,
+      limit,
+      count: data.length,
+      totalCount: parseInt(count as string),
+      totalPages: Math.ceil(parseInt(count as string) / limit),
+    };
+  }
+
+  /**
+   * Create batch of minute results for efficient bulk insertion
+   */
+  async createBatch(
+    videoId: number,
+    minuteResults: IVideoMinuteResultInput[],
+  ): Promise<IVideoMinuteResult[]> {
+    const batchData = minuteResults.map((result) =>
+      this.mapEntityToDbRow({
+        ...result,
+        videoId,
+      }),
+    );
+
+    const results = await this.knex
+      .batchInsert(this.tableName, batchData, 50)
+      .returning("*");
+    return this.mapDbRowsToEntities(results);
+  }
+
+  /**
+   * Delete all minute results for a specific video
+   */
+  async deleteByVideoId(videoId: number): Promise<boolean> {
+    const result = await this.knex(this.tableName)
+      .where("video_id", videoId)
+      .del();
+
+    return result > 0;
+  }
+
+  /**
+   * Get the minute range for a video (min and max minute numbers)
+   */
+  async getVideoMinuteRange(
+    videoId: number,
+  ): Promise<{ minMinute: number; maxMinute: number } | null> {
+    const result = await this.knex(this.tableName)
+      .where("video_id", videoId)
+      .min("minute_number as minMinute")
+      .max("minute_number as maxMinute")
+      .first();
+
+    if (!result || result.minMinute === null) {
+      return null;
+    }
+
+    return {
+      minMinute: result.minMinute,
+      maxMinute: result.maxMinute,
+    };
+  }
+
+  /**
+   * Map database row to entity (snake_case -> camelCase)
+   */
+  private mapDbRowToEntity(row: any): IVideoMinuteResult {
+    return {
+      id: row.id,
+      uuid: row.uuid,
+      videoId: row.video_id,
+      minuteNumber: row.minute_number,
+      results: row.results,
+      createdAt: row.created_at,
+      updatedAt: row.updated_at,
+    };
+  }
+
+  /**
+   * Map multiple database rows to entities
+   */
+  private mapDbRowsToEntities(rows: any[]): IVideoMinuteResult[] {
+    return rows.map((row) => this.mapDbRowToEntity(row));
+  }
+
+  /**
+   * Map entity to database row (camelCase -> snake_case)
+   */
+  private mapEntityToDbRow(
+    entity: Partial<IVideoMinuteResult | IVideoMinuteResultInput>,
+  ): any {
+    const dbRow: any = {};
+
+    if (entity.videoId !== undefined) dbRow.video_id = entity.videoId;
+    if ("minuteNumber" in entity && entity.minuteNumber !== undefined)
+      dbRow.minute_number = entity.minuteNumber;
+    if (entity.results !== undefined) dbRow.results = entity.results;
+    if ("createdAt" in entity && entity.createdAt !== undefined)
+      dbRow.created_at = entity.createdAt;
+    if ("updatedAt" in entity && entity.updatedAt !== undefined)
+      dbRow.updated_at = entity.updatedAt;
+
+    return dbRow;
+  }
+
+  /**
+   * Get minute results by video UUID (convenience method)
+   */
+  async getMinuteResultsByVideoUuid(
+    videoUuid: string,
+    startMinute?: number,
+    endMinute?: number,
+    page: number = 1,
+    limit: number = 100,
+  ): Promise<IDataPaginator<IVideoMinuteResult>> {
+    const offset = (page - 1) * limit;
+
+    const query = this.knex(this.tableName + " as vmr")
+      .innerJoin("video as v", "vmr.video_id", "v.id")
+      .select("vmr.*", "v.name as video_name", "v.uuid as video_uuid")
+      .where("v.uuid", videoUuid);
+
+    if (startMinute !== undefined) {
+      query.where("vmr.minute_number", ">=", startMinute);
+    }
+
+    if (endMinute !== undefined) {
+      query.where("vmr.minute_number", "<=", endMinute);
+    }
+
+    // Check if video exists and get count in single query
+    const [{ count }] = await query.clone().clearSelect().count("* as count");
+    const totalCount = parseInt(count as string);
+
+    if (totalCount === 0) {
+      // Check if video exists by trying to find it
+      const videoExists = await this.knex("video")
+        .where("uuid", videoUuid)
+        .first();
+      if (!videoExists) {
+        throw new Error(`Video with UUID ${videoUuid} not found`);
+      }
+    }
+
+    const data = await query
+      .orderBy("vmr.minute_number", "asc")
+      .limit(limit)
+      .offset(offset);
+
+    return {
+      success: true,
+      data: this.mapDbRowsToEntities(data),
+      page,
+      limit,
+      count: data.length,
+      totalCount,
+      totalPages: Math.ceil(totalCount / limit),
+    };
+  }
+}
+
+export default VideoMinuteResultDAO;