@pittica/google-business-intelligence-csv 1.2.0 → 1.4.0

package/dist/index.js

@@ -24,6 +24,21 @@ const {
 const {
   bootstrap
 } = require("./scripts/bootstrap");
+const {
+  upload
+} = require("./scripts/upload");
+const {
+  clean
+} = require("./scripts/clean");
+const {
+  pubSub
+} = require("./scripts/pub-sub");
+const {
+  rollback
+} = require("./scripts/rollback");
+const {
+  unlistened
+} = require("./scripts/unlistened");
 const {
   config
 } = require("./config");
@@ -32,5 +47,10 @@ exports.scripts = {
   bulk,
   day,
   file,
-  bootstrap
+  bootstrap,
+  upload,
+  clean,
+  pubSub,
+  rollback,
+  unlistened
 };

package/dist/scripts/bulk.js

@@ -38,16 +38,16 @@ const {
  * @param {string} temporary Temporary bucket.
  * @param {string} destination Destination bucket.
  */
-exports.bulk = async (source, temporary, destination) => {
+exports.bulk = async (source = null, temporary = null, destination = null) => {
   const storage = getStorage();
   const dataset = await getDataset(config.dataset.temporary.name, {
     location: config.dataset.temporary.location
   });
-  const bucketSource = await storage.bucket(source);
-  const bucketTemporary = await storage.bucket(temporary);
-  const bucketDestination = await storage.bucket(destination);
+  const bucketSource = await storage.bucket(source ?? config.bucket.upload);
+  const bucketTemporary = await storage.bucket(temporary ?? config.bucket.temporary);
+  const bucketDestination = await storage.bucket(destination ?? config.bucket.archive);
   const now = getNow();
-  storage.bucket(source).getFiles().then(response => {
+  bucketSource.getFiles().then(response => {
     log.info(`Starting bulk import...`);
     const files = extractStorageResponse(response, config.files.sql);
     process(dataset, bucketSource, bucketTemporary, bucketDestination, sort(files), now);

package/dist/scripts/clean.js

@@ -0,0 +1,23 @@
+// Copyright 2024-2026 Pittica S.r.l.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+
+const {
+  cleanDataset
+} = require("@pittica/google-bigquery-helpers");
+
+/**
+ * Drops the tables in temporary dataset.
+ *
+ * @returns {void}
+ */
+exports.clean = async () => await cleanDataset(`^${config.dataset.temporary.name}.*`);

package/dist/scripts/day.js

@@ -33,7 +33,8 @@ const {
   getSafeFilenameFromBucket,
   mergeFiledata,
   getNow,
-  formatDate
+  formatDate,
+  stringToDate
 } = require("@pittica/google-business-intelligence-helpers");
 const log = require("@pittica/logger-helpers");
 const {
@@ -51,12 +52,12 @@ const {
  * @param {string} temporary Temporary bucket.
  * @param {string} destination Destination bucket.
  */
-exports.day = async (date, source, temporary, destination) => {
+exports.day = async (date, source = null, temporary = null, destination = null) => {
   if (date) {
-    const day = formatDate(date);
+    const day = formatDate(typeof date === "string" ? stringToDate(date) : date);
     const now = getNow();
     log.info(`Starting "${day}" day import...`);
-    this.run(day, now, source, temporary, destination);
+    this.run(day, now, source ?? config.bucket.upload, temporary ?? config.bucket.temporary, destination ?? config.bucket.archive);
   }
 };
 

package/dist/scripts/file.js

@@ -46,14 +46,14 @@ const {
  * @param {boolean} clean A value indicating whether the temporary table will be deleted.
  * @returns {boolean} A value indicating whether the process has been done.
  */
-exports.file = async (name, bucketSource, bucketTemporary, bucketDestination, clean = true) => {
+exports.file = async (name, bucketSource = null, bucketTemporary = null, bucketDestination = null, clean = true) => {
   const filedata = splitName(name);
   if (isDataCsv(filedata, config.files.json)) {
     const storage = getStorage();
-    const source = storage.bucket(bucketSource).file(name);
+    const source = storage.bucket(bucketSource ?? config.bucket.upload).file(name);
     const [exists] = await source.exists();
     if (exists) {
-      return await copyFile(source, bucketTemporary).then(async response => {
+      return await copyFile(source, bucketTemporary ?? config.bucket.temporary).then(async response => {
         const file = copiedFile(response);
         if (file !== null) {
           deleteFile(source);
@@ -62,7 +62,7 @@ exports.file = async (name, bucketSource, bucketTemporary, bucketDestination, cl
           });
           const day = getNow();
           const table = await ds.table(getTemporaryTableName(filedata, config.dataset.temporary.prefix));
-          return await this.run(table, bucketDestination, file, filedata, day, clean);
+          return await this.run(table, bucketDestination ?? config.bucket.archive, file, filedata, day, clean);
         }
         return false;
       }).catch(logErrors);

package/dist/scripts/process.js

@@ -0,0 +1,20 @@
+// Copyright 2024-2026 Pittica S.r.l.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+/**
+ * Processes all CSV files in upload bucket bucket or the given file.
+ *
+ * @param {string} filename Element name.
+ */
+exports.process = async filename => filename ? await file(filename) : await bulk();

package/dist/scripts/pub-sub.js

@@ -0,0 +1,33 @@
+// Copyright 2024-2026 Pittica S.r.l.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+
+const {
+  bulk
+} = require("./bulk");
+
+/**
+ * Handles a Pub/Sub message event.
+ *
+ * @param {object} message Pub/Sub event data.
+ */
+exports.pubSub = async ({
+  data: {
+    message: {
+      data
+    }
+  }
+}) => {
+  if (data && Buffer.from(data, "base64").toString() == config.pubSub.message) {
+    await bulk(config.bucket.upload, config.bucket.temporary, config.bucket.archive);
+  }
+};

package/dist/scripts/rollback.js

@@ -0,0 +1,23 @@
+// Copyright 2024-2026 Pittica S.r.l.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+
+const {
+  moveFiles
+} = require("@pittica/google-cloud-storage-helpers");
+
+/**
+ * Moves the files from the temporary bucket to the source bucket.
+ *
+ * @returns {void}
+ */
+exports.rollback = async () => await moveFiles(config.bucket.temporary, config.bucket.upload);

package/dist/scripts/unlistened.js

@@ -0,0 +1,25 @@
+// Copyright 2024-2026 Pittica S.r.l.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+
+const {
+  scripts: {
+    unlistened
+  }
+} = require("@pittica/google-business-intelligence-helpers");
+
+/**
+ * Removes all not allowed files in "nempos-upload" bucket.
+ *
+ * @returns {void}
+ */
+exports.unlistened = async () => await unlistened(config.bucket.upload, config.files.sql);

package/dist/scripts/upload.js

@@ -0,0 +1,40 @@
+// Copyright 2024-2026 Pittica S.r.l.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+
+const {
+  splitName
+} = require("@pittica/google-business-intelligence-helpers");
+const {
+  day
+} = require("./day");
+
+/**
+ * Handles an "upload" event in Google Cloud Storage.
+ *
+ * @param {object} message Contains the event message which contains settings and type.
+ * @param {string} last Last uploaded file name without extension used as trigger.
+ */
+exports.upload = async ({
+  type,
+  data: {
+    bucket,
+    name
+  }
+}, last = "till_status") => {
+  if (type === "google.cloud.storage.object.v1.finalized" && bucket === config.bucket.upload) {
+    const file = splitName(name);
+    if (file.name === last) {
+      await day(file.date, config.bucket.upload, config.bucket.temporary, config.bucket.archive);
+    }
+  }
+};

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@pittica/google-business-intelligence-csv",
   "private": false,
-  "version": "1.2.0",
+  "version": "1.4.0",
   "description": "CSV importer for business intelligence for Google Cloud.",
   "main": "dist/index.js",
   "scripts": {