@firebaseextensions/firestore-bigquery-change-tracker 1.1.36 → 1.1.38

package/lib/bigquery/checkUpdates.d.ts

@@ -0,0 +1,19 @@
+import { Table, TableMetadata } from "@google-cloud/bigquery/build/src/table";
+import { FirestoreBigQueryEventHistoryTrackerConfig } from ".";
+interface TableRequiresUpdateOptions {
+    table: Table;
+    config: FirestoreBigQueryEventHistoryTrackerConfig;
+    documentIdColExists: boolean;
+    pathParamsColExists: boolean;
+    oldDataColExists: boolean;
+}
+export declare function tableRequiresUpdate({ table, config, documentIdColExists, pathParamsColExists, oldDataColExists, }: TableRequiresUpdateOptions): Promise<boolean>;
+interface ViewRequiresUpdateOptions {
+    metadata?: TableMetadata;
+    config: FirestoreBigQueryEventHistoryTrackerConfig;
+    documentIdColExists: boolean;
+    pathParamsColExists: boolean;
+    oldDataColExists: boolean;
+}
+export declare function viewRequiresUpdate({ metadata, config, documentIdColExists, pathParamsColExists, oldDataColExists, }: ViewRequiresUpdateOptions): boolean;
+export {};

package/lib/bigquery/clustering.d.ts

@@ -0,0 +1,14 @@
+import { FirestoreBigQueryEventHistoryTrackerConfig } from ".";
+import * as bigquery from "@google-cloud/bigquery";
+import { TableMetadata } from "@google-cloud/bigquery";
+export declare class Clustering {
+    config: FirestoreBigQueryEventHistoryTrackerConfig;
+    table: bigquery.Table;
+    schema: object;
+    constructor(config: FirestoreBigQueryEventHistoryTrackerConfig, table?: bigquery.Table, schema?: object);
+    hasValidTableReference(): boolean;
+    private hasInvalidFields;
+    private updateCluster;
+    private removeCluster;
+    updateClustering: (metaData: TableMetadata) => Promise<void>;
+}

package/lib/bigquery/handleFailedTransactions.d.ts

@@ -0,0 +1,3 @@
+import { FirestoreBigQueryEventHistoryTrackerConfig } from ".";
+declare const _default: (rows: any[], config: FirestoreBigQueryEventHistoryTrackerConfig, e: Error) => Promise<void>;
+export default _default;

package/lib/bigquery/index.d.ts

@@ -0,0 +1,77 @@
+import * as bigquery from "@google-cloud/bigquery";
+import { FirestoreEventHistoryTracker, FirestoreDocumentChangeEvent } from "../tracker";
+export { RawChangelogSchema, RawChangelogViewSchema } from "./schema";
+export interface FirestoreBigQueryEventHistoryTrackerConfig {
+    datasetId: string;
+    tableId: string;
+    datasetLocation?: string | undefined;
+    transformFunction?: string | undefined;
+    timePartitioning?: string | undefined;
+    timePartitioningField?: string | undefined;
+    timePartitioningFieldType?: string | undefined;
+    timePartitioningFirestoreField?: string | undefined;
+    clustering: string[] | null;
+    databaseId?: string | undefined;
+    wildcardIds?: boolean;
+    bqProjectId?: string | undefined;
+    backupTableId?: string | undefined;
+    useNewSnapshotQuerySyntax?: boolean;
+    skipInit?: boolean;
+    kmsKeyName?: string | undefined;
+}
+/**
+ * An FirestoreEventHistoryTracker that exports data to BigQuery.
+ *
+ * When the first event is received, it creates necessary BigQuery resources:
+ * - Dataset: {@link FirestoreBigQueryEventHistoryTrackerConfig#datasetId}.
+ * - Table: Raw change log table {@link FirestoreBigQueryEventHistoryTracker#rawChangeLogTableName}.
+ * - View: Latest view {@link FirestoreBigQueryEventHistoryTracker#rawLatestView}.
+ * If any subsequent data export fails, it will attempt to reinitialize.
+ */
+export declare class FirestoreBigQueryEventHistoryTracker implements FirestoreEventHistoryTracker {
+    config: FirestoreBigQueryEventHistoryTrackerConfig;
+    bq: bigquery.BigQuery;
+    _initialized: boolean;
+    constructor(config: FirestoreBigQueryEventHistoryTrackerConfig);
+    record(events: FirestoreDocumentChangeEvent[]): Promise<void>;
+    private transformRows;
+    serializeData(eventData: any): any;
+    /**
+     * Check whether a failed operation is retryable or not.
+     * Reasons for retrying:
+     * 1) We added a new column to our schema. Sometimes BQ is not ready to stream insertion records immediately
+     *    after adding a new column to an existing table (https://issuetracker.google.com/35905247)
+     */
+    private isRetryableInsertionError;
+    /**
+     * Tables can often take time to create and propagate.
+     * A half a second delay is added per check while the function
+     * continually re-checks until the referenced dataset and table become available.
+     */
+    private _waitForInitialization;
+    /**
+     * Inserts rows of data into the BigQuery raw change log table.
+     */
+    private insertData;
+    /**
+     * Creates the BigQuery resources with the expected schema for {@link FirestoreEventHistoryTracker}.
+     * After the first invokation, it skips initialization assuming these resources are still there.
+     */
+    initialize(): Promise<void>;
+    /**
+     * Creates the specified dataset if it doesn't already exists.
+     */
+    private initializeDataset;
+    /**
+     * Creates the raw change log table if it doesn't already exist.
+     */
+    private initializeRawChangeLogTable;
+    /**
+     * Creates the latest snapshot view, which returns only latest operations
+     * of all existing documents over the raw change log table.
+     */
+    private initializeLatestView;
+    bigqueryDataset(): bigquery.Dataset;
+    private rawChangeLogTableName;
+    private rawLatestView;
+}

package/lib/bigquery/partitioning.d.ts

@@ -0,0 +1,29 @@
+import { FirestoreBigQueryEventHistoryTrackerConfig } from ".";
+import { FirestoreDocumentChangeEvent } from "..";
+import * as bigquery from "@google-cloud/bigquery";
+export declare class Partitioning {
+    config: FirestoreBigQueryEventHistoryTrackerConfig;
+    table: bigquery.Table;
+    schema: object;
+    constructor(config: FirestoreBigQueryEventHistoryTrackerConfig, table?: bigquery.Table, schema?: object);
+    private isPartitioningEnabled;
+    private isValidPartitionTypeString;
+    private metaDataSchemaFields;
+    private isValidPartitionTypeDate;
+    private hasHourAndDatePartitionConfig;
+    private hasValidCustomPartitionConfig;
+    private hasValidTimePartitionOption;
+    private hasValidTimePartitionType;
+    hasExistingSchema(): Promise<boolean>;
+    hasValidTableReference(): boolean;
+    private isTablePartitioned;
+    isValidPartitionForExistingTable(): Promise<boolean>;
+    convertDateValue(fieldValue: Date): string;
+    getPartitionValue(event: FirestoreDocumentChangeEvent): {
+        [x: string]: any;
+    };
+    customFieldExists(fields?: any[]): boolean;
+    private shouldAddPartitioningToSchema;
+    addPartitioningToSchema(fields?: any[]): Promise<void>;
+    updateTableMetadata(options: bigquery.TableMetadata): Promise<void>;
+}

package/lib/bigquery/schema.d.ts

@@ -0,0 +1,56 @@
+import { FirestoreBigQueryEventHistoryTrackerConfig } from ".";
+export type BigQueryFieldMode = "NULLABLE" | "REPEATED" | "REQUIRED";
+export type BigQueryFieldType = "BOOLEAN" | "NUMERIC" | "RECORD" | "STRING" | "TIMESTAMP";
+export type BigQueryField = {
+    fields?: BigQueryField[];
+    mode: BigQueryFieldMode;
+    name: string;
+    type: BigQueryFieldType;
+};
+export declare const dataField: BigQueryField;
+export declare const documentNameField: BigQueryField;
+export declare const eventIdField: BigQueryField;
+export declare const operationField: BigQueryField;
+export declare const timestampField: BigQueryField;
+export declare const latitudeField: BigQueryField;
+export declare const longitudeField: BigQueryField;
+export declare const documentIdField: {
+    name: string;
+    mode: string;
+    type: string;
+    description: string;
+};
+export declare const documentPathParams: {
+    name: string;
+    mode: string;
+    type: string;
+    description: string;
+};
+export declare const oldDataField: {
+    name: string;
+    mode: string;
+    type: string;
+    description: string;
+};
+export declare const RawChangelogViewSchema: {
+    fields: {
+        name: string;
+        mode: string;
+        type: string;
+        description: string;
+    }[];
+};
+export declare const RawChangelogSchema: {
+    fields: {
+        name: string;
+        mode: string;
+        type: string;
+        description: string;
+    }[];
+};
+export declare const getNewPartitionField: (config: FirestoreBigQueryEventHistoryTrackerConfig) => {
+    name: string;
+    mode: string;
+    type: string;
+    description: string;
+};

package/lib/bigquery/snapshot.d.ts

@@ -0,0 +1,21 @@
+interface LatestConsistentSnapshotViewOptions {
+    datasetId: string;
+    tableName: string;
+    schema: any;
+    bqProjectId?: string;
+    useLegacyQuery?: boolean;
+}
+export declare const latestConsistentSnapshotView: ({ datasetId, tableName, schema, bqProjectId, useLegacyQuery, }: LatestConsistentSnapshotViewOptions) => {
+    query: string;
+    useLegacySql: boolean;
+};
+interface buildLatestSnapshotViewQueryOptions {
+    datasetId: string;
+    tableName: string;
+    timestampColumnName: string;
+    groupByColumns: string[];
+    bqProjectId?: string;
+    useLegacyQuery?: boolean;
+}
+export declare function buildLatestSnapshotViewQuery({ datasetId, tableName, timestampColumnName, groupByColumns, bqProjectId, useLegacyQuery, }: buildLatestSnapshotViewQueryOptions): string;
+export {};

package/lib/bigquery/utils.d.ts

@@ -0,0 +1,15 @@
+import { Dataset, Table } from "@google-cloud/bigquery";
+interface WaitForInitializationParams {
+    dataset: Dataset;
+    changelogName: string;
+}
+/**
+ * Periodically checks for the existence of a dataset and table until both are found or a maximum number of attempts is reached.
+ * @param {WaitForInitializationParams} params - Parameters for initialization including the dataset and the table name to check.
+ * @param {number} maxAttempts - Maximum number of attempts before giving up (defaults to 12).
+ * @returns {Promise<Table>} A promise that resolves with the Table if it exists, or rejects if it doesn't exist after maxAttempts or an error occurs.
+ * @throws {Error} Throws an error if the dataset or table cannot be verified to exist after multiple attempts or if an unexpected error occurs.
+ */
+export declare function waitForInitialization({ dataset, changelogName }: WaitForInitializationParams, maxAttempts?: number): Promise<Table>;
+export declare function parseErrorMessage(error: unknown, process?: string): string;
+export {};

package/lib/bigquery/validateProject.d.ts

@@ -0,0 +1 @@
+export declare const validateProject: (id: string) => Promise<boolean>;

package/lib/errors.d.ts

@@ -0,0 +1,2 @@
+export declare const changedFieldMode: (fieldName: string, bqMode: string, schemaMode: string) => Error;
+export declare const changedFieldType: (fieldName: string, bqType: string, schemaType: string) => Error;

package/lib/index.d.ts

@@ -0,0 +1,2 @@
+export { FirestoreBigQueryEventHistoryTracker, RawChangelogSchema, RawChangelogViewSchema, } from "./bigquery";
+export { ChangeType, FirestoreDocumentChangeEvent, FirestoreEventHistoryTracker, } from "./tracker";

package/lib/logs.d.ts

@@ -0,0 +1,56 @@
+import { Table } from "@google-cloud/bigquery";
+import { firestore } from "firebase-admin";
+export declare const arrayFieldInvalid: (fieldName: string) => void;
+export declare const bigQueryDatasetCreated: (datasetId: string) => void;
+export declare const bigQueryDatasetCreating: (datasetId: string) => void;
+export declare const bigQueryDatasetExists: (datasetId: string) => void;
+export declare const bigQueryErrorRecordingDocumentChange: (e: Error) => void;
+export declare const bigQueryLatestSnapshotViewQueryCreated: (query: string) => void;
+export declare const bigQuerySchemaViewCreated: (name: string) => void;
+export declare const bigQueryTableAlreadyExists: (tableName: string, datasetName: string) => void;
+export declare const bigQueryTableCreated: (tableName: string) => void;
+export declare const bigQueryTableCreating: (tableName: string) => void;
+export declare const bigQueryTableUpdated: (tableName: string) => void;
+export declare const bigQueryTableUpdating: (tableName: string) => void;
+export declare const bigQueryTableUpToDate: (tableName: string) => void;
+export declare const bigQueryTableValidated: (tableName: string) => void;
+export declare const bigQueryTableValidating: (tableName: string) => void;
+export declare const bigQueryUserDefinedFunctionCreating: (functionName: string) => void;
+export declare const bigQueryUserDefinedFunctionCreated: (functionName: string) => void;
+export declare const bigQueryViewCreated: (viewName: string) => void;
+export declare const bigQueryViewCreating: (viewName: string, query: string) => void;
+export declare const bigQueryViewAlreadyExists: (viewName: string, datasetName: string) => void;
+export declare const bigQueryViewUpdated: (viewName: string) => void;
+export declare const bigQueryViewUpdating: (viewName: string) => void;
+export declare const bigQueryViewUpToDate: (viewName: string) => void;
+export declare const bigQueryViewValidated: (viewName: string) => void;
+export declare const bigQueryViewValidating: (viewName: string) => void;
+export declare const complete: () => void;
+export declare const dataInserted: (rowCount: number) => void;
+export declare const dataInsertRetried: (rowCount: number) => void;
+export declare const dataInserting: (rowCount: number) => void;
+export declare const dataTypeInvalid: (fieldName: string, fieldType: string, dataType: string) => void;
+export declare const error: (err: Error) => void;
+export declare const timestampMissingValue: (fieldName: string) => void;
+export declare const addNewColumn: (table: string, field: string) => void;
+export declare const addPartitionFieldColumn: (table: any, field: any) => void;
+export declare const firestoreTimePartitionFieldError: (documentName: string | undefined, fieldName: string | undefined, firestoreFieldName: string | undefined, firestoreFieldData: any) => void;
+export declare const firestoreTimePartitioningParametersWarning: (fieldName: string | undefined, fieldType: string | undefined, firestoreFieldName: string | undefined, dataFirestoreField: firestore.Timestamp | string | undefined) => void;
+export declare const bigQueryTableInsertErrors: (insertErrors: [
+    {
+        row: object;
+        errors: Array<{
+            message: string;
+        }>;
+    }
+]) => void;
+export declare const updatedClustering: (fields: string) => void;
+export declare const removedClustering: (tableName: string) => void;
+export declare const cannotPartitionExistingTable: (table: Table) => void;
+export declare function invalidProjectIdWarning(bqProjectId: string): void;
+export declare function invalidTableReference(): void;
+export declare function hourAndDatePartitioningWarning(): void;
+export declare function invalidClustering(fields: string): void;
+export declare const tableCreationError: (table: any, message: any) => void;
+export declare const failedToInitializeWait: (message: any) => void;
+export declare const updatingMetadata: (tableName: any, resources: any) => void;

package/lib/tracker.d.ts

@@ -0,0 +1,23 @@
+export declare enum ChangeType {
+    CREATE = 0,
+    DELETE = 1,
+    UPDATE = 2,
+    IMPORT = 3
+}
+export interface FirestoreDocumentChangeEvent {
+    timestamp: string;
+    operation: ChangeType;
+    documentName: string;
+    eventId: string;
+    documentId: string;
+    pathParams?: {
+        documentId: string;
+        [key: string]: string;
+    } | null;
+    data: Object;
+    oldData?: Object | null;
+    useNewSnapshotQuerySyntax?: boolean | null;
+}
+export interface FirestoreEventHistoryTracker {
+    record(event: FirestoreDocumentChangeEvent[]): any;
+}

package/lib/types.d.ts

@@ -0,0 +1,5 @@
+export declare enum PartitionFieldType {
+    DATE = "DATE",
+    DATETIME = "DATETIME",
+    TIMESTAMP = "TIMESTAMP"
+}

package/package.json

@@ -5,7 +5,7 @@
     "url": "github.com/firebase/extensions.git",
     "directory": "firestore-bigquery-export/firestore-bigquery-change-tracker"
   },
-  "version": "1.1.36",
+  "version": "1.1.38",
   "description": "Core change-tracker library for Cloud Firestore Collection BigQuery Exports",
   "main": "./lib/index.js",
   "scripts": {
@@ -18,7 +18,9 @@
   },
   "files": [
     "lib/*.js",
-    "lib/bigquery/*.js"
+    "lib/bigquery/*.js",
+    "lib/*.d.ts",
+    "lib/bigquery/*.d.ts"
   ],
   "author": "Jan Wyszynski <wyszynski@google.com>",
   "license": "Apache-2.0",