@seeka-labs/cli-apps 3.7.0 → 3.8.2

package/dist/ai-context/internal/examples/v2/google-bigquery/app/server/src/lib/google-big-query/migrations/index.ts

@@ -0,0 +1,280 @@
+import { BigQuery } from '@google-cloud/bigquery';
+import * as fs from 'fs';
+import * as migrate from 'migrate';
+import * as path from 'path';
+import winston, { Logger } from 'winston';
+import { GoogleBigqueryAppInstallSettings } from '../../state/seeka/installations';
+import { isErrorDueToNoBillingOnProject } from './../utils';
+import { getMigrationState, setMigrationState } from './state';
+
+export interface MigrationResult {
+    success: boolean;
+    isErrorDueToNoBillingOnProject: boolean;
+    appliedMigrations: string[];
+    errors: string[];
+}
+
+export interface MigrationStatus {
+    id: string;
+    filename: string;
+    applied: boolean;
+    appliedAt?: Date;
+}
+
+export type BigQueryTableType = 'activities'
+
+/**
+ * Custom storage adapter for the migrate library that uses Redis
+ */
+class RedisMigrationStorage {
+    private logger: Logger;
+    private installationSettings: GoogleBigqueryAppInstallSettings;
+
+    constructor(logger: Logger, installationSettings: GoogleBigqueryAppInstallSettings) {
+        this.logger = logger;
+        this.installationSettings = installationSettings;
+    }
+
+    async load(cb: migrate.FileStoreLoadCallback) {
+        try {
+            const appliedMigrations = await getMigrationState(this.logger, this.installationSettings);
+
+            const migrations = appliedMigrations.map(migrationId => ({
+                title: migrationId,
+                description: migrationId,
+                timestamp: Date.now() // We don't store timestamps in Redis, so use current time
+            }));
+
+            const lastRun = appliedMigrations.length > 0 ? appliedMigrations[appliedMigrations.length - 1] : undefined;
+
+            this.logger.verbose('Loaded migration state from Redis', { migrations, lastRun });
+
+            cb(null, { lastRun, migrations })
+        } catch (error) {
+            this.logger.error('Failed to load migration state:', { ex: winston.exceptions.getAllInfo(error) });
+            cb(error, { migrations: [] })
+        }
+    }
+
+    async save(set: migrate.MigrationSet, cb: migrate.Callback) {
+        try {
+            // The migrate library will call this when migrations are applied
+            // We handle state management in the migration execution itself
+            this.logger.verbose('Migration state save called', { set });
+            cb(null)
+        } catch (error) {
+            this.logger.error('Failed to save migration state:', { ex: winston.exceptions.getAllInfo(error) });
+            cb(error)
+        }
+    }
+}
+
+/**
+ * Get the migrations directory path
+ */
+const getMigrationsDirectory = (table: BigQueryTableType): string => {
+    const theDir = path.join(__dirname, table);
+    return theDir;
+};
+
+/**
+ * Ensure migrations directory exists
+ */
+const ensureMigrationsDirectory = (table: BigQueryTableType, logger: Logger): void => {
+    const migrationsDir = getMigrationsDirectory(table);
+    if (!fs.existsSync(migrationsDir)) {
+        logger.warn('Creating migrations directory for ' + table);
+        fs.mkdirSync(migrationsDir, { recursive: true });
+    }
+};
+
+/**
+ * Execute a single SQL migration against BigQuery
+ */
+const executeSqlMigration = async (
+    logger: Logger,
+    bigquery: BigQuery,
+    migrationPath: string,
+    installationSettings: GoogleBigqueryAppInstallSettings
+): Promise<void> => {
+    const sql = fs.readFileSync(migrationPath, 'utf8');
+
+    // Replace placeholders in SQL with actual values
+    const processedSql = sql
+        .replace(/\${DATASET_ID}/g, installationSettings.bigQueryActivityDatasetId)
+        .replace(/\${TABLE_NAME}/g, installationSettings.bigQueryActivityTableName)
+        .replace(/\${PROJECT_ID}/g, installationSettings.googleProjectId);
+
+    logger.verbose(`Executing SQL migration:`, { migrationPath, sql: processedSql });
+
+    // Execute the migration SQL
+    const [job] = await bigquery.createQueryJob({
+        query: processedSql,
+        // location: 'US', // Adjust location as needed
+    });
+
+    // Wait for the job to complete
+    await job.getQueryResults();
+
+    logger.info(`Successfully executed SQL migration: ${path.basename(migrationPath)}`);
+};
+
+const handler = (name: string, installationSettings: GoogleBigqueryAppInstallSettings, bigquery: BigQuery, migrationsDir: string, isNewTable: boolean, logger: Logger): { up: Function, down: Function } => {
+    return {
+        up: async () => {
+            logger.info('up', name)
+
+            let success = false
+            try {
+                if (!isNewTable) {
+                    const migrationPath = path.join(migrationsDir, name);
+                    await executeSqlMigration(logger, bigquery, migrationPath, installationSettings);
+
+                    logger.info(`Applied migration`, { name });
+                }
+                else {
+                    logger.info(`Skipped migration due to new table already having schema`, { name });
+                }
+
+                success = true
+            } catch (error) {
+                const e = winston.exceptions.getAllInfo(error);
+                if ((error?.toString() as string).includes('Column already exists')) {
+                    // temp fix until we handle not applying migrations to newly created tables
+                    logger.warn("Failed to apply migration col already exists", { name, ex: e })
+                    success = true
+                }
+                else if (isErrorDueToNoBillingOnProject(e)) {
+                    logger.warn("Failed to apply migration due to billing error", { ex: e, name })
+                    // throw error
+                }
+                else {
+                    logger.error("Failed to apply migration", { ex: e, name })
+                    throw error
+                }
+            }
+
+            if (success) {
+                // Mark as applied in Redis
+                await setMigrationState(logger, installationSettings, name, true);
+                // result.appliedMigrations.push(migrationId);
+            }
+        },
+        down: () => {
+            logger.info('down', name)
+        }
+    }
+}
+
+/**
+ * Apply all pending migrations
+ */
+export const applyMigrations = async (
+    table: BigQueryTableType,
+    logger: Logger,
+    installationSettings: GoogleBigqueryAppInstallSettings,
+    isNewTable: boolean
+): Promise<MigrationResult> => {
+    const result: MigrationResult = {
+        success: true,
+        isErrorDueToNoBillingOnProject: false,
+        appliedMigrations: [],
+        errors: []
+    };
+
+    try {
+        ensureMigrationsDirectory(table, logger);
+
+        const bigquery = new BigQuery({
+            projectId: installationSettings.googleProjectId,
+            credentials: JSON.parse(installationSettings.googleServiceAccountJson)
+        });
+
+        const migrationsDir = getMigrationsDirectory(table);
+        const storage = new RedisMigrationStorage(logger, installationSettings);
+
+        const migrations: migrate.MigrationOptions['migrations'] = {
+            '001_addtimestamp.sql': handler('001_addtimestamp.sql', installationSettings, bigquery, migrationsDir, isNewTable, logger),
+            '002_addrowversion.sql': handler('002_addrowversion.sql', installationSettings, bigquery, migrationsDir, isNewTable, logger),
+        }
+
+        // Load migration set
+        const set: migrate.MigrationSet = await new Promise<any>((resolve, reject) => {
+            migrate.load({
+                stateStore: storage,
+                migrations,
+            }, (err, set) => {
+                if (err) {
+                    reject(err);
+                    throw err
+                }
+                set.up(function (err) {
+                    if (err) {
+                        reject(err);
+                        throw err
+                    }
+
+                    logger.info('migrations successfully ran')
+                    resolve(set)
+                })
+            });
+        });
+
+        logger.info('Migration complete', { s: set.lastRun, m: set.migrations });
+    } catch (error) {
+        const e = winston.exceptions.getAllInfo(error)
+        if (isErrorDueToNoBillingOnProject(e)) {
+            logger.warn('Migration process failed due to no billing', { ex: e });
+            // result.errors.push(`Migration process failed : ${(error as Error).message}`);
+            result.success = false;
+            result.isErrorDueToNoBillingOnProject = true
+        }
+        else {
+            logger.error('Migration process failed', { ex: e });
+            result.errors.push(`Migration process failed: ${(error as Error).message}`);
+            result.success = false;
+        }
+    }
+
+    return result;
+};
+
+/**
+ * Get migration status for all migrations
+ */
+// export const getMigrationStatus = async (
+//     table: BigQueryTableType,
+//     logger: Logger,
+//     installationSettings: GoogleBigqueryAppInstallSettings
+// ): Promise<MigrationStatus[]> => {
+//     try {
+//         ensureMigrationsDirectory(table, logger);
+
+//         const migrationsDir = getMigrationsDirectory(table);
+//         const appliedMigrations = await getMigrationState(logger, installationSettings);
+
+//         // Get all .sql files in migrations directory
+//         const migrationFiles = fs.readdirSync(migrationsDir)
+//             .filter(file => file.endsWith('.sql'))
+//             .sort();
+
+//         const status: MigrationStatus[] = migrationFiles.map(filename => {
+//             const id = filename.replace('.sql', '');
+//             const applied = appliedMigrations.includes(id);
+
+//             return {
+//                 id,
+//                 filename,
+//                 applied,
+//                 appliedAt: applied ? new Date() : undefined // We don't store timestamps, so use current time
+//             };
+//         });
+
+//         logger.info(`Retrieved status for ${status.length} migrations`);
+//         return status;
+
+//     } catch (error) {
+//         logger.error('Failed to get migration status:', { error: (error as Error).message });
+//         throw error;
+//     }
+// };

package/dist/ai-context/internal/examples/v2/google-bigquery/app/server/src/lib/google-big-query/migrations/state.ts

@@ -0,0 +1,98 @@
+import winston, { Logger } from 'winston';
+import { createRedisClient, getKey } from '../../state/redis';
+import { GoogleBigqueryAppInstallSettings } from '../../state/seeka/installations';
+
+const redisStateType = "bigquerymigrations"
+
+/**
+ * Redis key prefix for migration state
+ */
+const getMigrationStateKey = (installationSettings: GoogleBigqueryAppInstallSettings): string => {
+    // seeka:app:2ba125e93d2b4b9893b5772ae77973da:bigquerymigrations:937306fd-85c6-3989-0525-3a16a31928b9:seeka-application-test:seekalocaldev:stream-activity
+    // seeka:app:2ba125e93d2b4b9893b5772ae77973da:bigquerymigrations:seeka-application-test:seekalocaldev:stream-activity
+
+    return getKey(redisStateType, `${installationSettings.googleProjectId}:${installationSettings.bigQueryActivityDatasetId}:${installationSettings.bigQueryActivityTableName}`);
+};
+
+/**
+ * Get applied migrations from Redis
+ */
+export const getMigrationState = async (
+    logger: Logger,
+    installationSettings: GoogleBigqueryAppInstallSettings
+): Promise<string[]> => {
+    const client = createRedisClient();
+
+    try {
+        await client.connect();
+        const key = getMigrationStateKey(installationSettings);
+        const appliedMigrations = await client.sMembers(key);
+        const migrationsArray = Array.from(appliedMigrations).map(m => m.toString());
+
+        logger.verbose(`Retrieved ${migrationsArray.length} applied migrations from Redis`, {
+            key,
+            appliedMigrations: migrationsArray
+        });
+
+        return migrationsArray.sort(); // Return sorted for consistency
+    } catch (error) {
+        logger.error('Failed to get migration state from Redis:', { ex: winston.exceptions.getAllInfo(error) });
+        throw error;
+    } finally {
+        await client.destroy();
+    }
+};
+
+
+/**
+ * Set migration as applied in Redis
+ */
+export const setMigrationState = async (
+    logger: Logger,
+    installationSettings: GoogleBigqueryAppInstallSettings,
+    migrationId: string,
+    applied: boolean
+): Promise<void> => {
+    const client = createRedisClient();
+
+    try {
+        await client.connect();
+        const key = getMigrationStateKey(installationSettings);
+
+        if (applied) {
+            await client.sAdd(key, migrationId);
+            logger.verbose(`Marked migration ${migrationId} as applied in Redis`, { key });
+        } else {
+            await client.sRem(key, migrationId);
+            logger.verbose(`Marked migration ${migrationId} as not applied in Redis`, { key });
+        }
+    } catch (error) {
+        logger.error(`Failed to set migration state in Redis for ${migrationId}:`, { ex: winston.exceptions.getAllInfo(error) });
+        throw error;
+    } finally {
+        await client.destroy();
+    }
+};
+
+/**
+ * Clear all migration state (useful for testing or reset)
+ */
+export const clearMigrationState = async (
+    logger: Logger,
+    installationSettings: GoogleBigqueryAppInstallSettings
+): Promise<void> => {
+    const client = createRedisClient();
+
+    try {
+        await client.connect();
+        const key = getMigrationStateKey(installationSettings);
+        await client.del(key);
+
+        logger.info(`Cleared all migration state from Redis`, { key });
+    } catch (error) {
+        logger.error('Failed to clear migration state from Redis:', { ex: winston.exceptions.getAllInfo(error) });
+        throw error;
+    } finally {
+        await client.destroy();
+    }
+};

package/dist/ai-context/internal/examples/v2/google-bigquery/app/server/src/lib/google-big-query/models/index.ts

@@ -0,0 +1,15 @@
+
+import { SeekaActivityAcceptedWebhookContent, SeekaIdentityChangedWebhookContent } from '@seeka-labs/sdk-apps-server';
+
+export interface SeekaActivityBigQueryRecord extends BigQueryRecordBase, SeekaActivityAcceptedWebhookContent {
+
+}
+
+export interface SeekaIdentityChangedBigQueryRecord extends BigQueryRecordBase, SeekaIdentityChangedWebhookContent {
+
+}
+
+export interface BigQueryRecordBase {
+    timestamp?: Date
+    rowVersion?: number
+}

package/dist/ai-context/internal/examples/v2/google-bigquery/app/server/src/lib/google-big-query/schema/activities/index.ts

@@ -0,0 +1,86 @@
+import type bigquery from '@google-cloud/bigquery/build/src/types';
+import jsonSchemaBigquery from 'jsonschema-bigquery-custom';
+import type { Logger } from "winston";
+import * as winston from "winston";
+import { getSchemaByName, sanitiseBigQuerySchema } from "../";
+
+let activitySchemaCache: bigquery.ITableSchema | null = null;
+
+const generateActivityTableJsonSchema = async (logger: Logger): Promise<bigquery.ITableSchema | null> => {
+    try {
+        return getSchemaByName('SeekaActivityAcceptedWebhookContent', logger);
+    }
+    catch (err) {
+        logger.error('Error generating json schema', { ex: winston.exceptions.getAllInfo(err) });
+        return null;
+    }
+}
+
+export const activitySchemaFieldsToRemove = [
+    'identifiers.traits',
+    // 'identifiers.klaviyo.data',
+    'identifiers.klaviyo',
+    'activity.commerce.products.attributes',
+    'activity.contentBehaviour.changeProduct.product.attributes',
+    'activity.contentItem.items.attributes',
+];
+export const activityPayloadFieldsToRemove = [
+    'identifiers.traits',
+    // 'identifiers.klaviyo.data',
+    'identifiers.klaviyo',
+    'activity.commerce.products[].attributes',
+    'activity.contentBehaviour.changeProduct.product.attributes',
+    'activity.contentItem.items[].attributes',
+];
+export const activityPayloadFieldsToJsonStringify = [
+    'activity.properties'
+];
+export const getBigQueryActivityTableSchema = async (logger: Logger): Promise<bigquery.ITableSchema | null> => {
+    if (activitySchemaCache) return activitySchemaCache;
+
+    try {
+        const schema = await generateActivityTableJsonSchema(logger);
+        if (!schema) return null;
+
+        const bigquerySchema = (jsonSchemaBigquery as any).run(schema, {
+            preventAdditionalObjectProperties: false,
+            continueOnError: false
+        });
+        logger.verbose('Generated bigquery schema', { bigquerySchema: bigquerySchema.schema });
+
+        let sanitised = sanitiseBigQuerySchema(bigquerySchema.schema, activitySchemaFieldsToRemove, logger);
+        logger.verbose('Sanitised bigquery schema', { sanitised });
+
+        sanitised = extendBigQueryActivityStreamTableSchema(sanitised, logger);
+
+        activitySchemaCache = sanitised;
+
+        return sanitised
+    }
+    catch (err) {
+        logger.error('Error getting bigquery schema', { ex: winston.exceptions.getAllInfo(err) });
+        return null;
+    }
+}
+
+const extendBigQueryActivityStreamTableSchema = (schema: bigquery.ITableSchema, logger: Logger): bigquery.ITableSchema => {
+    if (!schema.fields) return schema;
+
+    // https://cloud.google.com/bigquery/docs/schemas#specifying_a_json_schema_file
+    // https://cloud.google.com/bigquery/docs/default-values
+    schema.fields.push({
+        name: "timestamp",
+        type: "TIMESTAMP",
+        mode: "NULLABLE",
+        description: "The timestamp for when the record was either created or last updated."
+    })
+
+    schema.fields.push({
+        name: "rowVersion",
+        type: "INT64",
+        mode: "NULLABLE",
+        description: "The version number of the row. Incremented by 1 for each update to the row."
+    })
+
+    return schema;
+}

package/dist/ai-context/internal/examples/v2/google-bigquery/app/server/src/lib/google-big-query/schema/index.test.ts

@@ -0,0 +1,208 @@
+import { afterAll, describe, expect, test } from '@jest/globals';
+import { sanitiseBigQueryRecord } from './';
+
+
+import { has } from 'lodash';
+import { logger } from '../../logging';
+import { activityPayloadFieldsToRemove } from './activities';
+
+const testPayload = {
+
+    activity: {
+
+        activityId: 'sh-35aa7bb4-C153-4D38-C381-9262B9A2C551',
+
+        activityName: 'addPaymentMethod',
+
+        commerce: {
+
+            checkoutIdentifier: 'bf0c08209b5c837338b59d0f36e2deeb',
+
+            currencyCode: 'AUD',
+
+            products: [
+
+                {
+
+                    attributes: {},
+
+                    brandName: 'Blootropic',
+
+                    categoryName: 'methylene-blue',
+
+                    currencyCode: 'AUD',
+
+                    lineItemIdentifier: '41446507479091',
+
+                    productIdentifier: '6965124530227',
+
+                    productName: 'Methylene Blue',
+
+                    quantity: 1,
+
+                    sku: 'MBAQ-100',
+
+                    type: 'singleVariant',
+
+                    unitPrice: 115,
+
+                    variantIdentifier: '41446507479091',
+
+                    variantName: '100ml / 3.4oz'
+                }
+            ],
+
+            totalPrice: 115
+        },
+
+        search: {},
+
+        time: '2024-06-20T12:39:30.067291+00:00'
+    },
+
+    identifiers: {
+
+        address: [
+
+            {
+
+                countryCode: 'US'
+            },
+
+            {
+
+                countryCode: 'AU',
+
+                locality: 'Scarborough',
+
+                postcode: '6019',
+
+                state: 'WA'
+            }
+        ],
+
+        bigCommerce: {},
+
+        dob: [],
+
+        facebook: {
+
+            fbp: [
+
+                'fb.1.1718886508831.5514781407'
+            ]
+        },
+
+        google: {
+
+            gclid: [
+
+                'EAIaIQobChMI5M_qlJfqhgMVWsVMAh2vhg_oEAQYCCABEgLat_D_BwE',
+
+                'EAIaIQobChMIk8G9-5jqhgMVBdsWBR25JAMxEAQYBSABEgJnr_D_BwE'
+            ],
+
+            userClientId: [
+
+                '38040646.1718886511',
+
+                '437043962.1718886511'
+            ]
+        },
+
+        ip: [
+
+            '2001:8004:6f20:e24e:55f6:a2fa:daff:f4d9'
+        ],
+
+        magento: {},
+
+        pinterest: {},
+
+        seekaPId: 'sk.1.1718886508829.7034966199',
+
+        shopify: {
+
+            cartToken: [
+
+                'Z2NwLWFzaWEtc291dGhlYXN0MTowMUowVFREOVNBRDdUUjA2QkFEM0owWkY4RA'
+            ],
+
+            checkoutToken: [
+
+                'bf0c08209b5c837338b59d0f36e2deeb'
+            ],
+
+            uniqueToken: [
+
+                'eb09256e-16b1-4fef-a0d1-2535beb116b3'
+            ]
+        },
+
+        snapchat: {
+
+            scid: [
+
+                'a832ac36-da54-4489-85d0-3fd79421ce40'
+            ]
+        },
+
+        tiktok: {
+
+            ttp: [
+
+                '-FlL091gWM_8yLHLKsDl8w9nIQS'
+            ]
+        },
+
+        ua: [
+
+            'Mozilla/5.0 (iPhone; CPU iPhone OS 16_7_8 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.6 Mobile/15E148 Safari/604.1'
+        ],
+
+        wooCommerce: {}
+    },
+
+    personId: 'sk.1.1718886508829.7034966199',
+
+    source: {
+
+        ip: '2001:8004:6f20:e24e:55f6:a2fa:daff:f4d9',
+
+        loc: 'https://store.blootropic.com/checkouts/cn/Z2NwLWFzaWEtc291dGhlYXN0MTowMUowVFREOVNBRDdUUjA2QkFEM0owWkY4RA/processing',
+
+        method: 'api/http/sdk/js/activity/send',
+
+        origin: 'browser',
+
+        runtime: {
+
+            client: {
+
+                type: 'shopify-pixel',
+
+                ver: '1.0.0'
+            },
+
+            type: 'js/@seeka-labs/converge',
+
+            ver: '1.7.15'
+        },
+
+        sess: '20240620.122828.5550345278',
+
+        time: '2024-06-20T12:39:30.0153877+00:00'
+    }
+}
+
+
+describe('sum module', () => {
+    test('adds 1 + 2 to equal 3', () => {
+        const sanitized = sanitiseBigQueryRecord(testPayload, activityPayloadFieldsToRemove, logger);
+        const p = 'activity.commerce.products[0].attributes';
+        expect(has(sanitized, p)).toBeFalsy();
+    });
+    afterAll(() => {
+        logger.close();
+    })
+});