@seeka-labs/cli-apps 3.8.7 → 3.8.10

package/dist/ai-context/internal/examples/v2/google-bigquery/app/server/src/lib/google-big-query/migrations/index.ts

@@ -1,280 +0,0 @@
-import { BigQuery } from '@google-cloud/bigquery';
-import * as fs from 'fs';
-import * as migrate from 'migrate';
-import * as path from 'path';
-import winston, { Logger } from 'winston';
-import { GoogleBigqueryAppInstallSettings } from '../../state/seeka/installations';
-import { isErrorDueToNoBillingOnProject } from './../utils';
-import { getMigrationState, setMigrationState } from './state';
-
-export interface MigrationResult {
-    success: boolean;
-    isErrorDueToNoBillingOnProject: boolean;
-    appliedMigrations: string[];
-    errors: string[];
-}
-
-export interface MigrationStatus {
-    id: string;
-    filename: string;
-    applied: boolean;
-    appliedAt?: Date;
-}
-
-export type BigQueryTableType = 'activities'
-
-/**
- * Custom storage adapter for the migrate library that uses Redis
- */
-class RedisMigrationStorage {
-    private logger: Logger;
-    private installationSettings: GoogleBigqueryAppInstallSettings;
-
-    constructor(logger: Logger, installationSettings: GoogleBigqueryAppInstallSettings) {
-        this.logger = logger;
-        this.installationSettings = installationSettings;
-    }
-
-    async load(cb: migrate.FileStoreLoadCallback) {
-        try {
-            const appliedMigrations = await getMigrationState(this.logger, this.installationSettings);
-
-            const migrations = appliedMigrations.map(migrationId => ({
-                title: migrationId,
-                description: migrationId,
-                timestamp: Date.now() // We don't store timestamps in Redis, so use current time
-            }));
-
-            const lastRun = appliedMigrations.length > 0 ? appliedMigrations[appliedMigrations.length - 1] : undefined;
-
-            this.logger.verbose('Loaded migration state from Redis', { migrations, lastRun });
-
-            cb(null, { lastRun, migrations })
-        } catch (error) {
-            this.logger.error('Failed to load migration state:', { ex: winston.exceptions.getAllInfo(error) });
-            cb(error, { migrations: [] })
-        }
-    }
-
-    async save(set: migrate.MigrationSet, cb: migrate.Callback) {
-        try {
-            // The migrate library will call this when migrations are applied
-            // We handle state management in the migration execution itself
-            this.logger.verbose('Migration state save called', { set });
-            cb(null)
-        } catch (error) {
-            this.logger.error('Failed to save migration state:', { ex: winston.exceptions.getAllInfo(error) });
-            cb(error)
-        }
-    }
-}
-
-/**
- * Get the migrations directory path
- */
-const getMigrationsDirectory = (table: BigQueryTableType): string => {
-    const theDir = path.join(__dirname, table);
-    return theDir;
-};
-
-/**
- * Ensure migrations directory exists
- */
-const ensureMigrationsDirectory = (table: BigQueryTableType, logger: Logger): void => {
-    const migrationsDir = getMigrationsDirectory(table);
-    if (!fs.existsSync(migrationsDir)) {
-        logger.warn('Creating migrations directory for ' + table);
-        fs.mkdirSync(migrationsDir, { recursive: true });
-    }
-};
-
-/**
- * Execute a single SQL migration against BigQuery
- */
-const executeSqlMigration = async (
-    logger: Logger,
-    bigquery: BigQuery,
-    migrationPath: string,
-    installationSettings: GoogleBigqueryAppInstallSettings
-): Promise<void> => {
-    const sql = fs.readFileSync(migrationPath, 'utf8');
-
-    // Replace placeholders in SQL with actual values
-    const processedSql = sql
-        .replace(/\${DATASET_ID}/g, installationSettings.bigQueryActivityDatasetId)
-        .replace(/\${TABLE_NAME}/g, installationSettings.bigQueryActivityTableName)
-        .replace(/\${PROJECT_ID}/g, installationSettings.googleProjectId);
-
-    logger.verbose(`Executing SQL migration:`, { migrationPath, sql: processedSql });
-
-    // Execute the migration SQL
-    const [job] = await bigquery.createQueryJob({
-        query: processedSql,
-        // location: 'US', // Adjust location as needed
-    });
-
-    // Wait for the job to complete
-    await job.getQueryResults();
-
-    logger.info(`Successfully executed SQL migration: ${path.basename(migrationPath)}`);
-};
-
-const handler = (name: string, installationSettings: GoogleBigqueryAppInstallSettings, bigquery: BigQuery, migrationsDir: string, isNewTable: boolean, logger: Logger): { up: Function, down: Function } => {
-    return {
-        up: async () => {
-            logger.info('up', name)
-
-            let success = false
-            try {
-                if (!isNewTable) {
-                    const migrationPath = path.join(migrationsDir, name);
-                    await executeSqlMigration(logger, bigquery, migrationPath, installationSettings);
-
-                    logger.info(`Applied migration`, { name });
-                }
-                else {
-                    logger.info(`Skipped migration due to new table already having schema`, { name });
-                }
-
-                success = true
-            } catch (error) {
-                const e = winston.exceptions.getAllInfo(error);
-                if ((error?.toString() as string).includes('Column already exists')) {
-                    // temp fix until we handle not applying migrations to newly created tables
-                    logger.warn("Failed to apply migration col already exists", { name, ex: e })
-                    success = true
-                }
-                else if (isErrorDueToNoBillingOnProject(e)) {
-                    logger.warn("Failed to apply migration due to billing error", { ex: e, name })
-                    // throw error
-                }
-                else {
-                    logger.error("Failed to apply migration", { ex: e, name })
-                    throw error
-                }
-            }
-
-            if (success) {
-                // Mark as applied in Redis
-                await setMigrationState(logger, installationSettings, name, true);
-                // result.appliedMigrations.push(migrationId);
-            }
-        },
-        down: () => {
-            logger.info('down', name)
-        }
-    }
-}
-
-/**
- * Apply all pending migrations
- */
-export const applyMigrations = async (
-    table: BigQueryTableType,
-    logger: Logger,
-    installationSettings: GoogleBigqueryAppInstallSettings,
-    isNewTable: boolean
-): Promise<MigrationResult> => {
-    const result: MigrationResult = {
-        success: true,
-        isErrorDueToNoBillingOnProject: false,
-        appliedMigrations: [],
-        errors: []
-    };
-
-    try {
-        ensureMigrationsDirectory(table, logger);
-
-        const bigquery = new BigQuery({
-            projectId: installationSettings.googleProjectId,
-            credentials: JSON.parse(installationSettings.googleServiceAccountJson)
-        });
-
-        const migrationsDir = getMigrationsDirectory(table);
-        const storage = new RedisMigrationStorage(logger, installationSettings);
-
-        const migrations: migrate.MigrationOptions['migrations'] = {
-            '001_addtimestamp.sql': handler('001_addtimestamp.sql', installationSettings, bigquery, migrationsDir, isNewTable, logger),
-            '002_addrowversion.sql': handler('002_addrowversion.sql', installationSettings, bigquery, migrationsDir, isNewTable, logger),
-        }
-
-        // Load migration set
-        const set: migrate.MigrationSet = await new Promise<any>((resolve, reject) => {
-            migrate.load({
-                stateStore: storage,
-                migrations,
-            }, (err, set) => {
-                if (err) {
-                    reject(err);
-                    throw err
-                }
-                set.up(function (err) {
-                    if (err) {
-                        reject(err);
-                        throw err
-                    }
-
-                    logger.info('migrations successfully ran')
-                    resolve(set)
-                })
-            });
-        });
-
-        logger.info('Migration complete', { s: set.lastRun, m: set.migrations });
-    } catch (error) {
-        const e = winston.exceptions.getAllInfo(error)
-        if (isErrorDueToNoBillingOnProject(e)) {
-            logger.warn('Migration process failed due to no billing', { ex: e });
-            // result.errors.push(`Migration process failed : ${(error as Error).message}`);
-            result.success = false;
-            result.isErrorDueToNoBillingOnProject = true
-        }
-        else {
-            logger.error('Migration process failed', { ex: e });
-            result.errors.push(`Migration process failed: ${(error as Error).message}`);
-            result.success = false;
-        }
-    }
-
-    return result;
-};
-
-/**
- * Get migration status for all migrations
- */
-// export const getMigrationStatus = async (
-//     table: BigQueryTableType,
-//     logger: Logger,
-//     installationSettings: GoogleBigqueryAppInstallSettings
-// ): Promise<MigrationStatus[]> => {
-//     try {
-//         ensureMigrationsDirectory(table, logger);
-
-//         const migrationsDir = getMigrationsDirectory(table);
-//         const appliedMigrations = await getMigrationState(logger, installationSettings);
-
-//         // Get all .sql files in migrations directory
-//         const migrationFiles = fs.readdirSync(migrationsDir)
-//             .filter(file => file.endsWith('.sql'))
-//             .sort();
-
-//         const status: MigrationStatus[] = migrationFiles.map(filename => {
-//             const id = filename.replace('.sql', '');
-//             const applied = appliedMigrations.includes(id);
-
-//             return {
-//                 id,
-//                 filename,
-//                 applied,
-//                 appliedAt: applied ? new Date() : undefined // We don't store timestamps, so use current time
-//             };
-//         });
-
-//         logger.info(`Retrieved status for ${status.length} migrations`);
-//         return status;
-
-//     } catch (error) {
-//         logger.error('Failed to get migration status:', { error: (error as Error).message });
-//         throw error;
-//     }
-// };

package/dist/ai-context/internal/examples/v2/google-bigquery/app/server/src/lib/google-big-query/migrations/state.ts

@@ -1,98 +0,0 @@
-import winston, { Logger } from 'winston';
-import { createRedisClient, getKey } from '../../state/redis';
-import { GoogleBigqueryAppInstallSettings } from '../../state/seeka/installations';
-
-const redisStateType = "bigquerymigrations"
-
-/**
- * Redis key prefix for migration state
- */
-const getMigrationStateKey = (installationSettings: GoogleBigqueryAppInstallSettings): string => {
-    // seeka:app:2ba125e93d2b4b9893b5772ae77973da:bigquerymigrations:937306fd-85c6-3989-0525-3a16a31928b9:seeka-application-test:seekalocaldev:stream-activity
-    // seeka:app:2ba125e93d2b4b9893b5772ae77973da:bigquerymigrations:seeka-application-test:seekalocaldev:stream-activity
-
-    return getKey(redisStateType, `${installationSettings.googleProjectId}:${installationSettings.bigQueryActivityDatasetId}:${installationSettings.bigQueryActivityTableName}`);
-};
-
-/**
- * Get applied migrations from Redis
- */
-export const getMigrationState = async (
-    logger: Logger,
-    installationSettings: GoogleBigqueryAppInstallSettings
-): Promise<string[]> => {
-    const client = createRedisClient();
-
-    try {
-        await client.connect();
-        const key = getMigrationStateKey(installationSettings);
-        const appliedMigrations = await client.sMembers(key);
-        const migrationsArray = Array.from(appliedMigrations).map(m => m.toString());
-
-        logger.verbose(`Retrieved ${migrationsArray.length} applied migrations from Redis`, {
-            key,
-            appliedMigrations: migrationsArray
-        });
-
-        return migrationsArray.sort(); // Return sorted for consistency
-    } catch (error) {
-        logger.error('Failed to get migration state from Redis:', { ex: winston.exceptions.getAllInfo(error) });
-        throw error;
-    } finally {
-        await client.destroy();
-    }
-};
-
-
-/**
- * Set migration as applied in Redis
- */
-export const setMigrationState = async (
-    logger: Logger,
-    installationSettings: GoogleBigqueryAppInstallSettings,
-    migrationId: string,
-    applied: boolean
-): Promise<void> => {
-    const client = createRedisClient();
-
-    try {
-        await client.connect();
-        const key = getMigrationStateKey(installationSettings);
-
-        if (applied) {
-            await client.sAdd(key, migrationId);
-            logger.verbose(`Marked migration ${migrationId} as applied in Redis`, { key });
-        } else {
-            await client.sRem(key, migrationId);
-            logger.verbose(`Marked migration ${migrationId} as not applied in Redis`, { key });
-        }
-    } catch (error) {
-        logger.error(`Failed to set migration state in Redis for ${migrationId}:`, { ex: winston.exceptions.getAllInfo(error) });
-        throw error;
-    } finally {
-        await client.destroy();
-    }
-};
-
-/**
- * Clear all migration state (useful for testing or reset)
- */
-export const clearMigrationState = async (
-    logger: Logger,
-    installationSettings: GoogleBigqueryAppInstallSettings
-): Promise<void> => {
-    const client = createRedisClient();
-
-    try {
-        await client.connect();
-        const key = getMigrationStateKey(installationSettings);
-        await client.del(key);
-
-        logger.info(`Cleared all migration state from Redis`, { key });
-    } catch (error) {
-        logger.error('Failed to clear migration state from Redis:', { ex: winston.exceptions.getAllInfo(error) });
-        throw error;
-    } finally {
-        await client.destroy();
-    }
-};

package/dist/ai-context/internal/examples/v2/google-bigquery/app/server/src/lib/google-big-query/models/index.ts

@@ -1,15 +0,0 @@
-
-import { SeekaActivityAcceptedWebhookContent, SeekaIdentityChangedWebhookContent } from '@seeka-labs/sdk-apps-server';
-
-export interface SeekaActivityBigQueryRecord extends BigQueryRecordBase, SeekaActivityAcceptedWebhookContent {
-
-}
-
-export interface SeekaIdentityChangedBigQueryRecord extends BigQueryRecordBase, SeekaIdentityChangedWebhookContent {
-
-}
-
-export interface BigQueryRecordBase {
-    timestamp?: Date
-    rowVersion?: number
-}

package/dist/ai-context/internal/examples/v2/google-bigquery/app/server/src/lib/google-big-query/schema/activities/index.ts

@@ -1,86 +0,0 @@
-import type bigquery from '@google-cloud/bigquery/build/src/types';
-import jsonSchemaBigquery from 'jsonschema-bigquery-custom';
-import type { Logger } from "winston";
-import * as winston from "winston";
-import { getSchemaByName, sanitiseBigQuerySchema } from "../";
-
-let activitySchemaCache: bigquery.ITableSchema | null = null;
-
-const generateActivityTableJsonSchema = async (logger: Logger): Promise<bigquery.ITableSchema | null> => {
-    try {
-        return getSchemaByName('SeekaActivityAcceptedWebhookContent', logger);
-    }
-    catch (err) {
-        logger.error('Error generating json schema', { ex: winston.exceptions.getAllInfo(err) });
-        return null;
-    }
-}
-
-export const activitySchemaFieldsToRemove = [
-    'identifiers.traits',
-    // 'identifiers.klaviyo.data',
-    'identifiers.klaviyo',
-    'activity.commerce.products.attributes',
-    'activity.contentBehaviour.changeProduct.product.attributes',
-    'activity.contentItem.items.attributes',
-];
-export const activityPayloadFieldsToRemove = [
-    'identifiers.traits',
-    // 'identifiers.klaviyo.data',
-    'identifiers.klaviyo',
-    'activity.commerce.products[].attributes',
-    'activity.contentBehaviour.changeProduct.product.attributes',
-    'activity.contentItem.items[].attributes',
-];
-export const activityPayloadFieldsToJsonStringify = [
-    'activity.properties'
-];
-export const getBigQueryActivityTableSchema = async (logger: Logger): Promise<bigquery.ITableSchema | null> => {
-    if (activitySchemaCache) return activitySchemaCache;
-
-    try {
-        const schema = await generateActivityTableJsonSchema(logger);
-        if (!schema) return null;
-
-        const bigquerySchema = (jsonSchemaBigquery as any).run(schema, {
-            preventAdditionalObjectProperties: false,
-            continueOnError: false
-        });
-        logger.verbose('Generated bigquery schema', { bigquerySchema: bigquerySchema.schema });
-
-        let sanitised = sanitiseBigQuerySchema(bigquerySchema.schema, activitySchemaFieldsToRemove, logger);
-        logger.verbose('Sanitised bigquery schema', { sanitised });
-
-        sanitised = extendBigQueryActivityStreamTableSchema(sanitised, logger);
-
-        activitySchemaCache = sanitised;
-
-        return sanitised
-    }
-    catch (err) {
-        logger.error('Error getting bigquery schema', { ex: winston.exceptions.getAllInfo(err) });
-        return null;
-    }
-}
-
-const extendBigQueryActivityStreamTableSchema = (schema: bigquery.ITableSchema, logger: Logger): bigquery.ITableSchema => {
-    if (!schema.fields) return schema;
-
-    // https://cloud.google.com/bigquery/docs/schemas#specifying_a_json_schema_file
-    // https://cloud.google.com/bigquery/docs/default-values
-    schema.fields.push({
-        name: "timestamp",
-        type: "TIMESTAMP",
-        mode: "NULLABLE",
-        description: "The timestamp for when the record was either created or last updated."
-    })
-
-    schema.fields.push({
-        name: "rowVersion",
-        type: "INT64",
-        mode: "NULLABLE",
-        description: "The version number of the row. Incremented by 1 for each update to the row."
-    })
-
-    return schema;
-}

package/dist/ai-context/internal/examples/v2/google-bigquery/app/server/src/lib/google-big-query/schema/index.test.ts

@@ -1,208 +0,0 @@
-import { afterAll, describe, expect, test } from '@jest/globals';
-import { sanitiseBigQueryRecord } from './';
-
-
-import { has } from 'lodash';
-import { logger } from '../../logging';
-import { activityPayloadFieldsToRemove } from './activities';
-
-const testPayload = {
-
-    activity: {
-
-        activityId: 'sh-35aa7bb4-C153-4D38-C381-9262B9A2C551',
-
-        activityName: 'addPaymentMethod',
-
-        commerce: {
-
-            checkoutIdentifier: 'bf0c08209b5c837338b59d0f36e2deeb',
-
-            currencyCode: 'AUD',
-
-            products: [
-
-                {
-
-                    attributes: {},
-
-                    brandName: 'Blootropic',
-
-                    categoryName: 'methylene-blue',
-
-                    currencyCode: 'AUD',
-
-                    lineItemIdentifier: '41446507479091',
-
-                    productIdentifier: '6965124530227',
-
-                    productName: 'Methylene Blue',
-
-                    quantity: 1,
-
-                    sku: 'MBAQ-100',
-
-                    type: 'singleVariant',
-
-                    unitPrice: 115,
-
-                    variantIdentifier: '41446507479091',
-
-                    variantName: '100ml / 3.4oz'
-                }
-            ],
-
-            totalPrice: 115
-        },
-
-        search: {},
-
-        time: '2024-06-20T12:39:30.067291+00:00'
-    },
-
-    identifiers: {
-
-        address: [
-
-            {
-
-                countryCode: 'US'
-            },
-
-            {
-
-                countryCode: 'AU',
-
-                locality: 'Scarborough',
-
-                postcode: '6019',
-
-                state: 'WA'
-            }
-        ],
-
-        bigCommerce: {},
-
-        dob: [],
-
-        facebook: {
-
-            fbp: [
-
-                'fb.1.1718886508831.5514781407'
-            ]
-        },
-
-        google: {
-
-            gclid: [
-
-                'EAIaIQobChMI5M_qlJfqhgMVWsVMAh2vhg_oEAQYCCABEgLat_D_BwE',
-
-                'EAIaIQobChMIk8G9-5jqhgMVBdsWBR25JAMxEAQYBSABEgJnr_D_BwE'
-            ],
-
-            userClientId: [
-
-                '38040646.1718886511',
-
-                '437043962.1718886511'
-            ]
-        },
-
-        ip: [
-
-            '2001:8004:6f20:e24e:55f6:a2fa:daff:f4d9'
-        ],
-
-        magento: {},
-
-        pinterest: {},
-
-        seekaPId: 'sk.1.1718886508829.7034966199',
-
-        shopify: {
-
-            cartToken: [
-
-                'Z2NwLWFzaWEtc291dGhlYXN0MTowMUowVFREOVNBRDdUUjA2QkFEM0owWkY4RA'
-            ],
-
-            checkoutToken: [
-
-                'bf0c08209b5c837338b59d0f36e2deeb'
-            ],
-
-            uniqueToken: [
-
-                'eb09256e-16b1-4fef-a0d1-2535beb116b3'
-            ]
-        },
-
-        snapchat: {
-
-            scid: [
-
-                'a832ac36-da54-4489-85d0-3fd79421ce40'
-            ]
-        },
-
-        tiktok: {
-
-            ttp: [
-
-                '-FlL091gWM_8yLHLKsDl8w9nIQS'
-            ]
-        },
-
-        ua: [
-
-            'Mozilla/5.0 (iPhone; CPU iPhone OS 16_7_8 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.6 Mobile/15E148 Safari/604.1'
-        ],
-
-        wooCommerce: {}
-    },
-
-    personId: 'sk.1.1718886508829.7034966199',
-
-    source: {
-
-        ip: '2001:8004:6f20:e24e:55f6:a2fa:daff:f4d9',
-
-        loc: 'https://store.blootropic.com/checkouts/cn/Z2NwLWFzaWEtc291dGhlYXN0MTowMUowVFREOVNBRDdUUjA2QkFEM0owWkY4RA/processing',
-
-        method: 'api/http/sdk/js/activity/send',
-
-        origin: 'browser',
-
-        runtime: {
-
-            client: {
-
-                type: 'shopify-pixel',
-
-                ver: '1.0.0'
-            },
-
-            type: 'js/@seeka-labs/converge',
-
-            ver: '1.7.15'
-        },
-
-        sess: '20240620.122828.5550345278',
-
-        time: '2024-06-20T12:39:30.0153877+00:00'
-    }
-}
-
-
-describe('sum module', () => {
-    test('adds 1 + 2 to equal 3', () => {
-        const sanitized = sanitiseBigQueryRecord(testPayload, activityPayloadFieldsToRemove, logger);
-        const p = 'activity.commerce.products[0].attributes';
-        expect(has(sanitized, p)).toBeFalsy();
-    });
-    afterAll(() => {
-        logger.close();
-    })
-});