firestore-meilisearch 0.1.0

package/functions/lib/meilisearch/create-index.js

@@ -0,0 +1,17 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.initMeilisearchIndex = void 0;
+const meilisearch_1 = require("meilisearch");
+/**
+ * initMeilisearchIndex
+ * @param {MeilisearchConfig} - Meilisearch configuration
+ * @return {Index}
+ */
+function initMeilisearchIndex({ host, apiKey, indexUid, }) {
+    const client = new meilisearch_1.MeiliSearch({
+        host,
+        apiKey,
+    });
+    return client.index(indexUid);
+}
+exports.initMeilisearchIndex = initMeilisearchIndex;

package/functions/lib/meilisearch-index.js

@@ -0,0 +1,17 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.createMeiliSearchIndex = void 0;
+const meilisearch_1 = require("meilisearch");
+/**
+ * createMeiliSearchIndex
+ * @param {MeiliSearchConfig}
+ * @return {Index}
+ */
+function createMeiliSearchIndex({ host, apiKey, indexUid, }) {
+    const client = new meilisearch_1.MeiliSearch({
+        host,
+        apiKey,
+    });
+    return client.index(indexUid);
+}
+exports.createMeiliSearchIndex = createMeiliSearchIndex;

package/functions/lib/types.js

@@ -0,0 +1,2 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });

package/functions/lib/util.js

@@ -0,0 +1,47 @@
+'use strict';
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.getFieldsToIndex = exports.getChangedDocumentId = exports.getChangeType = exports.ChangeType = void 0;
+const config_1 = require("./config");
+var ChangeType;
+(function (ChangeType) {
+    ChangeType[ChangeType["CREATE"] = 0] = "CREATE";
+    ChangeType[ChangeType["DELETE"] = 1] = "DELETE";
+    ChangeType[ChangeType["UPDATE"] = 2] = "UPDATE";
+})(ChangeType = exports.ChangeType || (exports.ChangeType = {}));
+/**
+ * Get type of the modification perform on a document.
+ * @param {Change<T>} change The Functions interface for events that change state.
+ * @return {ChangeType} Final type of the event.
+ */
+function getChangeType(change) {
+    if (!change.after.exists) {
+        return ChangeType.DELETE;
+    }
+    if (!change.before.exists) {
+        return ChangeType.CREATE;
+    }
+    return ChangeType.UPDATE;
+}
+exports.getChangeType = getChangeType;
+/**
+ * Get final id of a document after modification.
+ * @param {Change<T>} change The Functions interface for events that change state.
+ * @return {string} Final state type of the event.
+ */
+function getChangedDocumentId(change) {
+    if (change.after.exists) {
+        return change.after.id;
+    }
+    return change.before.id;
+}
+exports.getChangedDocumentId = getChangedDocumentId;
+/**
+ * Returns the MEILISEARCH_FIELDS_TO_INDEX value from the config file and formats it.
+ * @return {string[]} An array of fields.
+ */
+function getFieldsToIndex() {
+    return config_1.config.meilisearch.fieldsToIndex
+        ? config_1.config.meilisearch.fieldsToIndex.split(/[ ,]+/)
+        : [];
+}
+exports.getFieldsToIndex = getFieldsToIndex;

package/functions/lib/version.js

@@ -0,0 +1,4 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.version = void 0;
+exports.version = '0.1.0';

package/functions/package.json

@@ -0,0 +1,53 @@
+{
+  "name": "firestore-meilisearch",
+  "version": "0.1.0",
+  "scripts": {
+    "lint": "eslint .",
+    "lint:fix": "eslint . --fix",
+    "build": "tsc",
+    "watch": "tsc --watch",
+    "playground": "nodemon ./lib/import --project 521120192778 --source-collection-path movies --index movies --batch-size 300 --non-interactive -H 'http://localhost:7700' -a masterKey",
+    "serve": "yarn build && firebase emulators:start --only functions",
+    "shell": "yarn build && firebase functions:shell",
+    "start": "yarn shell",
+    "deploy": "firebase deploy --only functions",
+    "logs": "firebase functions:log",
+    "test": "jest",
+    "test:watch": "jest --watch",
+    "test:coverage": "jest --coverage"
+  },
+  "engines": {
+    "node": ">=14.0.0"
+  },
+  "main": "lib/index.js",
+  "dependencies": {
+    "firebase-admin": "^10.0.2",
+    "firebase-functions": "^3.16.0",
+    "meilisearch": "^0.25.0"
+  },
+  "devDependencies": {
+    "@babel/preset-typescript": "^7.15.0",
+    "@types/jest": "^27.0.2",
+    "@typescript-eslint/eslint-plugin": "^4.32.0",
+    "@typescript-eslint/parser": "^4.32.0",
+    "commander": "^8.3.0",
+    "eslint": "^7.6.0",
+    "eslint-config-google": "^0.14.0",
+    "eslint-config-prettier": "^8.3.0",
+    "eslint-plugin-import": "^2.24.2",
+    "eslint-plugin-jest": "^24.4.2",
+    "eslint-plugin-prettier": "^4.0.0",
+    "firebase-functions-test": "^0.3.2",
+    "inquirer": "^8.2.0",
+    "jest": "^27.2.2",
+    "jest-mock": "^27.1.1",
+    "js-yaml": "^4.1.0",
+    "mocked-env": "^1.3.5",
+    "prettier": "^2.4.1",
+    "ts-jest": "^27.0.5",
+    "ts-node": "^10.2.1",
+    "typescript": "^4.4.3"
+  },
+  "private": true,
+  "bin": "lib/import/index.js"
+}

package/functions/src/adapter.ts

@@ -0,0 +1,106 @@
+'use strict'
+/*
+ * Copyright 2022 Meilisearch
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { DocumentSnapshot } from 'firebase-functions/lib/providers/firestore'
+import {
+  DocumentReference,
+  GeoPoint,
+  Timestamp,
+} from 'firebase-admin/lib/firestore'
+import { getFieldsToIndex } from './util'
+import * as logs from './logs'
+
+type MeilisearchGeoPoint = {
+  lat: number
+  lng: number
+}
+
+type FirestoreRow =
+  | null
+  | boolean
+  | number
+  | string
+  | DocumentReference
+  | GeoPoint
+  | Timestamp
+  | Array<any>
+  | Map<any, any>
+  | MeilisearchGeoPoint
+
+/**
+ * Adapts documents from the Firestore database to Meilisearch compatible documents.
+ * @param {string} documentId Document id.
+ * @param {DocumentSnapshot} snapshot Snapshot of the data contained in the document read from your Firestore database.
+ * @return {Record<string, any>} A properly formatted document to be added or updated in Meilisearch.
+ */
+export function adaptDocument(
+  documentId: string,
+  snapshot: DocumentSnapshot
+): Record<string, any> {
+  const fields = getFieldsToIndex()
+  const data = snapshot.data() || {}
+  if ('_firestore_id' in data) {
+    delete data.id
+  }
+  if (fields.length === 0) {
+    return { _firestore_id: documentId, ...data }
+  }
+  const document = Object.keys(data).reduce(
+    (acc, key) => {
+      if (fields.includes(key)) {
+        const [field, value] = adaptValues(key, data[key])
+        return { ...acc, [field]: value }
+      }
+      return acc
+    },
+    { _firestore_id: documentId }
+  )
+  return document
+}
+
+/**
+ * Checks and adapts each values to be compatible with Meilisearch documents.
+ * @param {string} field
+ * @param {FirestoreRow} value
+ * @return {[string,FirestoreRow]} A properly formatted array of field and value.
+ */
+export function adaptValues(
+  field: string,
+  value: FirestoreRow
+): [string, FirestoreRow | MeilisearchGeoPoint] {
+  if (value instanceof GeoPoint) {
+    if (field === '_geo') {
+      logs.infoGeoPoint(true)
+      return [field, adaptGeoPoint(value)]
+    } else {
+      logs.infoGeoPoint(false)
+    }
+  }
+  return [field, value]
+}
+
+/**
+ * Adapts GeoPoint Firestore instance to fit with Meilisearch geo point.
+ * @param {GeoPoint} geoPoint GeoPoint Firestore object.
+ * @return {MeilisearchGeoPoint} A properly formatted geo point for Meilisearch.
+ */
+const adaptGeoPoint = (geoPoint: GeoPoint): MeilisearchGeoPoint => {
+  return {
+    lat: geoPoint.latitude,
+    lng: geoPoint.longitude,
+  }
+}

package/functions/src/config.ts

@@ -0,0 +1,34 @@
+'use strict'
+/*
+ * Copyright 2022 Meilisearch
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import { MeilisearchConfig } from './types'
+
+type PluginConfiguration = {
+  location: string
+  collectionPath: string
+  meilisearch: MeilisearchConfig
+}
+
+export const config: PluginConfiguration = {
+  location: process.env.LOCATION || 'europe-west1',
+  collectionPath: process.env.COLLECTION_PATH || '',
+  meilisearch: {
+    host: process.env.MEILISEARCH_HOST || '',
+    apiKey: process.env.MEILISEARCH_API_KEY || '',
+    indexUid: process.env.MEILISEARCH_INDEX_NAME || '',
+    fieldsToIndex: process.env.MEILISEARCH_FIELDS_TO_INDEX || '',
+  },
+}

package/functions/src/import/config.ts

@@ -0,0 +1,207 @@
+import { Command } from 'commander'
+import * as inquirer from 'inquirer'
+import { MeilisearchConfig } from '../types'
+
+const FIRESTORE_VALID_CHARACTERS = /^[^/]+$/
+const FIRESTORE_COLLECTION_NAME_MAX_CHARS = 6144
+const PROJECT_ID_MAX_CHARS = 30
+const MEILISEARCH_VALID_CHARACTERS = /^[a-zA-Z-_0-9,]*$/
+const MEILISEARCH_UID_MAX_CHARS = 6144
+
+// Create a new interface to manage the arguments from the non-interactive command line.
+const program = new Command()
+
+program
+  .name('firestore-meilisearch')
+  .option(
+    '--non-interactive',
+    'Parse all input from command line flags instead of prompting the caller.',
+    false
+  )
+  .option(
+    '-P, --project <project>',
+    'Firebase Project ID for project containing the Cloud Firestore database.'
+  )
+  .option(
+    '-s, --source-collection-path <source-collection-path>',
+    'The path of the Cloud Firestore Collection to import from. (This may, or may not, be the same Collection for which you plan to mirror changes.)'
+  )
+  .option(
+    '-q, --query-collection-group [true|false]',
+    "Use 'true' for a collection group query, otherwise a collection query is performed."
+  )
+  .option(
+    '-i, --index <index>',
+    "The Uid of the index in Meilisearch to import to. (An index will be created if it doesn't already exist.)"
+  )
+  .option(
+    '-b, --batch-size [batch-size]',
+    'Number of documents to stream into Meilisearch at once.',
+    value => parseInt(value, 10),
+    1000
+  )
+  .option(
+    '-H, --host <host>',
+    'The Host of your Meilisearch database. Example: http://localhost:7700.'
+  )
+  .option(
+    '-a, --api-key <api-key>',
+    'The Meilisearch API key with permission to perform actions on indexes. Both the private key and the master key are valid choices but we strongly recommend using the private key for security purposes.'
+  )
+
+const validateInput = (
+  value: string,
+  name: string,
+  regex: RegExp,
+  sizeLimit: number
+) => {
+  if (!value || typeof value !== 'string' || value.trim() === '') {
+    return `Please supply a ${name}`
+  }
+  if (value.trim().length > sizeLimit) {
+    return `${name} must be at most ${sizeLimit} characters long`
+  }
+  if (!value.match(regex)) {
+    return `The ${name} does not match the regular expression provided`
+  }
+  return true
+}
+
+const validateBatchSize = (value: string) => {
+  if (/^\d+$/.test(value) == false) {
+    return `The batchsize ${value} should be a number`
+  }
+  return parseInt(value, 10) > 0
+}
+
+// Questions for the interactive user interface in command line.
+const questions = [
+  {
+    message: 'What is your Firebase project ID?',
+    name: 'project',
+    type: 'input',
+    validate: value =>
+      validateInput(
+        value,
+        'project ID',
+        FIRESTORE_VALID_CHARACTERS,
+        PROJECT_ID_MAX_CHARS
+      ),
+  },
+  {
+    message:
+      'What is the path of the Cloud Firestore Collection you would like to import from? ' +
+      '(This may, or may not, be the same Collection for which you plan to mirror changes.)',
+    name: 'sourceCollectionPath',
+    type: 'input',
+    validate: value =>
+      validateInput(
+        value,
+        'collection path',
+        FIRESTORE_VALID_CHARACTERS,
+        FIRESTORE_COLLECTION_NAME_MAX_CHARS
+      ),
+  },
+  {
+    message: 'Would you like to import documents via a Collection Group query?',
+    name: 'queryCollectionGroup',
+    type: 'confirm',
+    default: false,
+  },
+  {
+    message:
+      "What is the Uid of the Meilisearch index that you would like to use? (A index will be created if it doesn't already exist)",
+    name: 'index',
+    type: 'input',
+    validate: value =>
+      validateInput(
+        value,
+        'index',
+        MEILISEARCH_VALID_CHARACTERS,
+        MEILISEARCH_UID_MAX_CHARS
+      ),
+  },
+  {
+    message:
+      'How many documents should the import stream into Meilisearch at once?',
+    name: 'batchSize',
+    type: 'input',
+    default: 1000,
+    validate: validateBatchSize,
+  },
+  {
+    message:
+      'What is the host of the Meilisearch database that you would like to use? Example: http://localhost:7700.',
+    name: 'host',
+    type: 'input',
+  },
+  {
+    message:
+      'Which Meilisearch API key with permission to perform actions on indexes do you like to use? Both the private key and the master key are valid choices but we strongly recommend using the private key for security purposes.',
+    name: 'apiKey',
+    type: 'input',
+  },
+]
+
+export interface CLIConfig {
+  projectId: string
+  sourceCollectionPath: string
+  queryCollectionGroup: boolean
+  batchSize: string
+  meilisearch: MeilisearchConfig
+}
+
+/**
+ * Parse the argument from the interactive or non-interactive command line.
+ */
+export async function parseConfig(): Promise<CLIConfig> {
+  program.parse(process.argv)
+
+  const options = program.opts()
+  if (options.nonInteractive) {
+    if (
+      options.project === undefined ||
+      options.sourceCollectionPath === undefined ||
+      options.index === undefined ||
+      options.host === undefined ||
+      options.apiKey === undefined ||
+      !validateBatchSize(options.batchSize)
+    ) {
+      program.outputHelp()
+      process.exit(1)
+    }
+
+    return {
+      projectId: options.project,
+      sourceCollectionPath: options.sourceCollectionPath,
+      queryCollectionGroup: options.queryCollectionGroup === 'true',
+      batchSize: options.batchSize,
+      meilisearch: {
+        indexUid: options.index,
+        host: options.host,
+        apiKey: options.apiKey,
+      },
+    }
+  }
+  const {
+    project,
+    sourceCollectionPath,
+    queryCollectionGroup,
+    index,
+    batchSize,
+    host,
+    apiKey,
+  } = await inquirer.prompt(questions)
+
+  return {
+    projectId: project,
+    sourceCollectionPath: sourceCollectionPath,
+    queryCollectionGroup: queryCollectionGroup,
+    batchSize: batchSize,
+    meilisearch: {
+      indexUid: index,
+      host: host,
+      apiKey: apiKey,
+    },
+  }
+}

package/functions/src/import/index.ts

@@ -0,0 +1,115 @@
+#!/usr/bin/env node
+
+/*
+ * Copyright 2022 Meilisearch
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import * as admin from 'firebase-admin'
+import { DocumentSnapshot } from 'firebase-functions/lib/providers/firestore'
+import { CLIConfig, parseConfig } from './config'
+import * as logs from '../logs'
+import { adaptDocument } from '../adapter'
+import { initMeilisearchIndex } from '../meilisearch/create-index'
+import { Index } from '../types'
+
+const run = async () => {
+  // Retrieve all arguments from the commande line.
+  const config: CLIConfig = await parseConfig()
+
+  // Initialize Firebase using the Google Credentials in the GOOGLE_APPLICATION_CREDENTIALS environment variable.
+  admin.initializeApp({
+    credential: admin.credential.applicationDefault(),
+    databaseURL: `https://${config.projectId}.firebaseio.com`,
+  })
+
+  const database = admin.firestore()
+
+  // Initialize Meilisearch index.
+  const index = initMeilisearchIndex(config.meilisearch)
+
+  await retrieveCollectionFromFirestore(database, config, index)
+}
+
+/**
+ * Retrieves a collection or collection group in Firestore and aggregates the data.
+ * @param {FirebaseFirestore.Firestore} database
+ * @param {CLIConfig} config
+ * @param {Index} index
+ */
+async function retrieveCollectionFromFirestore(
+  database: FirebaseFirestore.Firestore,
+  config: CLIConfig,
+  index: Index
+): Promise<number> {
+  const batch: number = parseInt(config.batchSize)
+
+  let query
+  let total = 0
+  let batches = 0
+  let lastDocument = null
+  let lastBatchSize: number = batch
+
+  while (lastBatchSize === batch) {
+    batches++
+
+    if (config.queryCollectionGroup) {
+      query = database.collectionGroup(config.sourceCollectionPath).limit(batch)
+    } else {
+      query = database.collection(config.sourceCollectionPath).limit(batch)
+    }
+
+    if (lastDocument !== null) {
+      query = query.startAfter(lastDocument)
+    }
+
+    const snapshot = await query.get()
+    const docs = snapshot.docs
+
+    if (docs.length === 0) break
+    total += await sendDocumentsToMeilisearch(docs, index)
+
+    if (docs.length) {
+      lastDocument = docs[docs.length - 1]
+    }
+    lastBatchSize = docs.length
+  }
+
+  logs.importData(total, batches)
+  return total
+}
+
+/**
+ * Adapts documents and indexes them in Meilisearch.
+ * @param {any} docs
+ * @param {Index} index
+ * @param {Change<DocumentSnapshot>} change
+ */
+async function sendDocumentsToMeilisearch(
+  docs: DocumentSnapshot[],
+  index: Index
+): Promise<number> {
+  const document = docs.map(snapshot => {
+    return adaptDocument(snapshot.id, snapshot)
+  })
+  try {
+    await index.addDocuments(document, { primaryKey: '_firestore_id' })
+  } catch (e) {
+    logs.error(e as Error)
+    return 0
+  }
+  return document.length
+}
+
+void run()