firestore-meilisearch 0.1.9 → 0.1.11

package/README.md

@@ -48,7 +48,7 @@ Documents indexed in Meilisearch must have an [unique id](https://docs.meilisear
 
 **Important:**  If your documents contain a field called `_firestore_id`, it will be ignored.
 
-If you are using `GeoPoint`, the field should be named `_geo` to be recognized by Meilisearch for [geosearch]((https://docs.meilisearch.com/reference/features/geosearch.html#geosearch)).
+If you are using `GeoPoint`, the field should be named `_geo` to be recognized by Meilisearch for [geosearch](https://docs.meilisearch.com/reference/features/geosearch.html#geosearch).
 
 #### Backfill your Meilisearch data
 
@@ -102,7 +102,7 @@ All Firebase services offer a free tier of usage.
 
 [![Install this extension in your Firebase project](https://www.gstatic.com/mobilesdk/210513_mobilesdk/install-extension.png "Install this extension in your Firebase project")][install-link]
 
-[install-link]: https://console.firebase.google.com/project/_/extensions/install?ref=meilisearch/firestore-meilisearch
+[install-link]: https://extensions.dev/extensions/meilisearch/firestore-meilisearch
 
 ### Firebase CLI
 
@@ -118,7 +118,7 @@ firebase ext:install meilisearch/firestore-meilisearch --project=[your-project-i
 
 ## 🤖 Compatibility with Meilisearch
 
-This package only guarantees the compatibility with the [version v0.29.0 of Meilisearch](https://github.com/meilisearch/meilisearch/releases/tag/v0.29.0).
+This package only guarantees the compatibility with the [version v0.30.0 of Meilisearch](https://github.com/meilisearch/meilisearch/releases/tag/v0.30.0).
 
 ## ⚙️ Development Workflow and Contributing
 

package/__tests__/adapter.test.ts

@@ -1,41 +1,33 @@
 import * as firebaseFunctionsTestInit from 'firebase-functions-test'
-import { mocked } from 'ts-jest/utils'
 import { mockConsoleInfo } from './__mocks__/console'
 import { firestore } from 'firebase-admin/lib/firestore'
 import { adaptDocument, adaptValues } from '../src/adapter'
-import { getFieldsToIndex } from '../src/util'
 import defaultDocument from './data/document'
 
-jest.mock('../src/util')
-
 // Mocking of Firebase functions
 const firebaseMock = firebaseFunctionsTestInit()
 
 describe('extensions process', () => {
   describe('adaptDocument', () => {
-    const mockedGetFieldsToIndex = mocked(getFieldsToIndex, true)
-
     test('adaptDocument with fields not set', () => {
-      mockedGetFieldsToIndex.mockReturnValueOnce([])
       const snapshot = firebaseMock.firestore.makeDocumentSnapshot(
         defaultDocument.document,
         `docs/${defaultDocument.id}`
       )
 
-      expect(adaptDocument(defaultDocument.id, snapshot)).toStrictEqual({
+      expect(adaptDocument(defaultDocument.id, snapshot, '')).toStrictEqual({
         _firestore_id: defaultDocument.id,
         ...defaultDocument.document,
       })
     })
 
     test('adaptDocument with fields not set and with id field in document', () => {
-      mockedGetFieldsToIndex.mockReturnValueOnce([])
       const snapshot = firebaseMock.firestore.makeDocumentSnapshot(
         { id: '12345', ...defaultDocument.document },
         `docs/${defaultDocument.id}`
       )
 
-      expect(adaptDocument(defaultDocument.id, snapshot)).toStrictEqual({
+      expect(adaptDocument(defaultDocument.id, snapshot, '')).toStrictEqual({
         _firestore_id: defaultDocument.id,
         id: '12345',
         ...defaultDocument.document,
@@ -43,17 +35,18 @@ describe('extensions process', () => {
     })
 
     test('adaptDocument with fields set', () => {
-      mockedGetFieldsToIndex.mockReturnValueOnce([
-        'title',
-        'overview',
-        'release_date',
-      ])
       const snapshot = firebaseMock.firestore.makeDocumentSnapshot(
         defaultDocument.document,
         `docs/${defaultDocument.id}`
       )
 
-      expect(adaptDocument(defaultDocument.id, snapshot)).toStrictEqual({
+      expect(
+        adaptDocument(
+          defaultDocument.id,
+          snapshot,
+          'title,overview,release_date'
+        )
+      ).toStrictEqual({
         _firestore_id: defaultDocument.id,
         title: defaultDocument.document.title,
         overview: defaultDocument.document.overview,

package/__tests__/util.test.ts

@@ -134,7 +134,7 @@ describe('getChangedDocumentId', () => {
 describe('getFieldsToIndex', () => {
   let util
   let restoreEnv
-  let mockGetFieldsToIndex
+  let mockParseFieldsToIndex
   const config = global.config
 
   beforeEach(() => {
@@ -150,48 +150,32 @@ describe('getFieldsToIndex', () => {
 
   test('return empty list', () => {
     util = require('../src/util')
-    mockGetFieldsToIndex = util.getFieldsToIndex()
-    expect(mockGetFieldsToIndex).toMatchObject([])
+    mockParseFieldsToIndex = util.parseFieldsToIndex()
+    expect(mockParseFieldsToIndex).toMatchObject([])
   })
 
   test('return list with one field', () => {
-    restoreEnv = mockedEnv({
-      ...defaultEnvironment,
-      MEILISEARCH_FIELDS_TO_INDEX: 'field',
-    })
     util = require('../src/util')
-    mockGetFieldsToIndex = util.getFieldsToIndex()
-    expect(mockGetFieldsToIndex).toMatchObject(['field'])
+    mockParseFieldsToIndex = util.parseFieldsToIndex('field')
+    expect(mockParseFieldsToIndex).toMatchObject(['field'])
   })
 
   test('return list with multiple fields', () => {
-    restoreEnv = mockedEnv({
-      ...defaultEnvironment,
-      MEILISEARCH_FIELDS_TO_INDEX: 'field1,field2,field3',
-    })
     util = require('../src/util')
-    mockGetFieldsToIndex = util.getFieldsToIndex()
-    expect(mockGetFieldsToIndex).toMatchObject(['field1', 'field2', 'field3'])
+    mockParseFieldsToIndex = util.parseFieldsToIndex('field1,field2,field3')
+    expect(mockParseFieldsToIndex).toMatchObject(['field1', 'field2', 'field3'])
   })
 
   test('return list with multiple fields and spaces', () => {
-    restoreEnv = mockedEnv({
-      ...defaultEnvironment,
-      MEILISEARCH_FIELDS_TO_INDEX: 'field1, field2,  field3',
-    })
     util = require('../src/util')
-    mockGetFieldsToIndex = util.getFieldsToIndex()
-    expect(mockGetFieldsToIndex).toMatchObject(['field1', 'field2', 'field3'])
+    mockParseFieldsToIndex = util.parseFieldsToIndex('field1, field2,  field3')
+    expect(mockParseFieldsToIndex).toMatchObject(['field1', 'field2', 'field3'])
   })
 
   test('return list of fiels with underscore', () => {
-    restoreEnv = mockedEnv({
-      ...defaultEnvironment,
-      MEILISEARCH_FIELDS_TO_INDEX: 'field_1,field_2,field_3',
-    })
     util = require('../src/util')
-    mockGetFieldsToIndex = util.getFieldsToIndex()
-    expect(mockGetFieldsToIndex).toMatchObject([
+    mockParseFieldsToIndex = util.parseFieldsToIndex('field_1,field_2,field_3')
+    expect(mockParseFieldsToIndex).toMatchObject([
       'field_1',
       'field_2',
       'field_3',

package/lib/adapter.js

@@ -8,25 +8,17 @@ const logs = require("./logs");
  * Adapts documents from the Firestore database to Meilisearch compatible documents.
  * @param {string} documentId Document id.
  * @param {DocumentSnapshot} snapshot Snapshot of the data contained in the document read from your Firestore database.
+ * @param {string} fieldsToIndex list of fields added in the document send to Meilisearch.
  * @return {Record<string, any>} A properly formatted document to be added or updated in Meilisearch.
  */
-function adaptDocument(documentId, snapshot) {
-    const fields = (0, util_1.getFieldsToIndex)();
+function adaptDocument(documentId, snapshot, fieldsToIndex) {
+    const fields = (0, util_1.parseFieldsToIndex)(fieldsToIndex);
     const data = snapshot.data() || {};
     if ('_firestore_id' in data) {
         delete data.id;
     }
-    if (fields.length === 0) {
-        return { _firestore_id: documentId, ...data };
-    }
-    const document = Object.keys(data).reduce((acc, key) => {
-        if (fields.includes(key)) {
-            const [field, value] = adaptValues(key, data[key]);
-            return { ...acc, [field]: value };
-        }
-        return acc;
-    }, { _firestore_id: documentId });
-    return document;
+    const document = (0, util_1.sanitizeDocuments)(fields, data);
+    return { _firestore_id: documentId, ...document };
 }
 exports.adaptDocument = adaptDocument;
 /**

package/lib/import/config.js

@@ -19,7 +19,8 @@ program
     .option('-i, --index <index>', "The Uid of the index in Meilisearch to import to. (An index will be created if it doesn't already exist.)")
     .option('-b, --batch-size [batch-size]', 'Number of documents to stream into Meilisearch at once.', value => parseInt(value, 10), 1000)
     .option('-H, --host <host>', 'The Host of your Meilisearch database. Example: http://localhost:7700.')
-    .option('-a, --api-key <api-key>', 'The Meilisearch API key with permission to perform actions on indexes. Both the private key and the master key are valid choices but we strongly recommend using the private key for security purposes.');
+    .option('-a, --api-key <api-key>', 'The Meilisearch API key with permission to perform actions on indexes. Both the private key and the master key are valid choices but we strongly recommend using the private key for security purposes.')
+    .option('-f, --fields-to-index <fields-to-index>', 'list of fields added in the document send to Meilisearch. Default contains all fields.');
 const validateInput = (value, name, regex, sizeLimit) => {
     if (!value || typeof value !== 'string' || value.trim() === '') {
         return `Please supply a ${name}`;
@@ -82,6 +83,12 @@ const questions = [
         name: 'apiKey',
         type: 'input',
     },
+    {
+        message: ' What fields do you want to index in Meilisearch? Create a comma-separated list of the field names, or leave it blank to include all fields. The id field is always indexed even when omitted from the list.',
+        name: 'fieldsToIndex',
+        default: '*',
+        type: 'input',
+    },
 ];
 /**
  * Parse the argument from the interactive or non-interactive command line.
@@ -108,10 +115,11 @@ async function parseConfig() {
                 indexUid: options.index,
                 host: options.host,
                 apiKey: options.apiKey,
+                fieldsToIndex: options.fieldsToIndex,
             },
         };
     }
-    const { project, sourceCollectionPath, queryCollectionGroup, index, batchSize, host, apiKey, } = await inquirer.prompt(questions);
+    const { project, sourceCollectionPath, queryCollectionGroup, index, batchSize, host, apiKey, fieldsToIndex, } = await inquirer.prompt(questions);
     return {
         projectId: project,
         sourceCollectionPath: sourceCollectionPath,
@@ -121,6 +129,7 @@ async function parseConfig() {
             indexUid: index,
             host: host,
             apiKey: apiKey,
+            fieldsToIndex: fieldsToIndex,
         },
     };
 }

package/lib/import/index.js

@@ -41,6 +41,7 @@ const run = async () => {
  * @param {Index} index
  */
 async function retrieveCollectionFromFirestore(database, config, index) {
+    var _a;
     const batch = parseInt(config.batchSize);
     let query;
     let total = 0;
@@ -62,7 +63,7 @@ async function retrieveCollectionFromFirestore(database, config, index) {
         const docs = snapshot.docs;
         if (docs.length === 0)
             break;
-        total += await sendDocumentsToMeilisearch(docs, index);
+        total += await sendDocumentsToMeilisearch(docs, index, ((_a = config.meilisearch) === null || _a === void 0 ? void 0 : _a.fieldsToIndex) || '');
         if (docs.length) {
             lastDocument = docs[docs.length - 1];
         }
@@ -73,13 +74,13 @@ async function retrieveCollectionFromFirestore(database, config, index) {
 }
 /**
  * Adapts documents and indexes them in Meilisearch.
- * @param {any} docs
+ * @param {Change<DocumentSnapshot>} docs
  * @param {Index} index
- * @param {Change<DocumentSnapshot>} change
+ * @param {string} fieldsToIndex list of fields added in the document send to Meilisearch.
  */
-async function sendDocumentsToMeilisearch(docs, index) {
+async function sendDocumentsToMeilisearch(docs, index, fieldsToIndex) {
     const document = docs.map(snapshot => {
-        return (0, adapter_1.adaptDocument)(snapshot.id, snapshot);
+        return (0, adapter_1.adaptDocument)(snapshot.id, snapshot, fieldsToIndex);
     });
     try {
         await index.addDocuments(document, { primaryKey: '_firestore_id' });

package/lib/index.js

@@ -56,7 +56,7 @@ async function handleAddDocument(documentId, snapshot) {
     try {
         logs.addDocument(documentId);
         if ((0, validate_1.validateDocumentId)(documentId)) {
-            const document = (0, adapter_1.adaptDocument)(documentId, snapshot);
+            const document = (0, adapter_1.adaptDocument)(documentId, snapshot, config_1.config.meilisearch.fieldsToIndex || '');
             const { taskUid } = await index.addDocuments([document], {
                 primaryKey: '_firestore_id',
             });
@@ -98,7 +98,7 @@ async function handleUpdateDocument(documentId, after) {
     try {
         logs.updateDocument(documentId);
         if ((0, validate_1.validateDocumentId)(documentId)) {
-            const document = (0, adapter_1.adaptDocument)(documentId, after);
+            const document = (0, adapter_1.adaptDocument)(documentId, after, config_1.config.meilisearch.fieldsToIndex || '');
             const { taskUid } = await index.addDocuments([document]);
             firebase_functions_1.logger.info(`Document update request for document with ID ${documentId} added to task list (task ID ${taskUid}).`);
         }

package/lib/util.js

@@ -1,7 +1,6 @@
 'use strict';
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.getFieldsToIndex = exports.getChangedDocumentId = exports.getChangeType = exports.ChangeType = void 0;
-const config_1 = require("./config");
+exports.sanitizeDocuments = exports.parseFieldsToIndex = exports.getChangedDocumentId = exports.getChangeType = exports.ChangeType = void 0;
 var ChangeType;
 (function (ChangeType) {
     ChangeType[ChangeType["CREATE"] = 0] = "CREATE";
@@ -36,12 +35,35 @@ function getChangedDocumentId(change) {
 }
 exports.getChangedDocumentId = getChangedDocumentId;
 /**
- * Returns the MEILISEARCH_FIELDS_TO_INDEX value from the config file and formats it.
+ * Parse the fieldsToIndex string into an array.
+ *
+ * @param  {string} fieldsToIndex
  * @return {string[]} An array of fields.
  */
-function getFieldsToIndex() {
-    return config_1.config.meilisearch.fieldsToIndex
-        ? config_1.config.meilisearch.fieldsToIndex.split(/[ ,]+/)
-        : [];
+function parseFieldsToIndex(fieldsToIndex) {
+    return fieldsToIndex ? fieldsToIndex.split(/[ ,]+/) : [];
 }
-exports.getFieldsToIndex = getFieldsToIndex;
+exports.parseFieldsToIndex = parseFieldsToIndex;
+/**
+ * Remove unwanted fields from the document before it is send to Meilisearch.
+ *
+ * @param  {string[]} fieldsToIndex
+ * @param  {Record<string, any>} document
+ * @return {Record<string, any>} sanitized document
+ *
+ */
+function sanitizeDocuments(fieldsToIndex, document) {
+    if (fieldsToIndex.length === 0) {
+        return document;
+    }
+    if (fieldsToIndex.includes('*')) {
+        return document;
+    }
+    for (const key in document) {
+        if (!fieldsToIndex.includes(key)) {
+            delete document[key];
+        }
+    }
+    return document;
+}
+exports.sanitizeDocuments = sanitizeDocuments;

package/lib/version.js

@@ -1,4 +1,4 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.version = void 0;
-exports.version = '0.1.9';
+exports.version = '0.1.11';

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "firestore-meilisearch",
-  "version": "0.1.9",
+  "version": "0.1.11",
   "scripts": {
     "lint": "eslint .",
     "lint:fix": "eslint . --fix",
@@ -25,7 +25,7 @@
     "firebase-admin": "^9.8.0",
     "firebase-functions": "^3.16.0",
     "inquirer": "^8.2.2",
-    "meilisearch": "^0.28.0"
+    "meilisearch": "^0.30.0"
   },
   "devDependencies": {
     "@babel/preset-typescript": "^7.15.0",

package/src/adapter.ts

@@ -17,7 +17,7 @@
 
 import { DocumentSnapshot } from 'firebase-functions/lib/providers/firestore'
 import { firestore } from 'firebase-admin/lib/firestore'
-import { getFieldsToIndex } from './util'
+import { parseFieldsToIndex, sanitizeDocuments } from './util'
 import * as logs from './logs'
 
 type MeilisearchGeoPoint = {
@@ -41,31 +41,24 @@ type FirestoreRow =
  * Adapts documents from the Firestore database to Meilisearch compatible documents.
  * @param {string} documentId Document id.
  * @param {DocumentSnapshot} snapshot Snapshot of the data contained in the document read from your Firestore database.
+ * @param {string} fieldsToIndex list of fields added in the document send to Meilisearch.
  * @return {Record<string, any>} A properly formatted document to be added or updated in Meilisearch.
  */
 export function adaptDocument(
   documentId: string,
-  snapshot: DocumentSnapshot
+  snapshot: DocumentSnapshot,
+  fieldsToIndex: string
 ): Record<string, any> {
-  const fields = getFieldsToIndex()
+  const fields = parseFieldsToIndex(fieldsToIndex)
+
   const data = snapshot.data() || {}
   if ('_firestore_id' in data) {
     delete data.id
   }
-  if (fields.length === 0) {
-    return { _firestore_id: documentId, ...data }
-  }
-  const document = Object.keys(data).reduce(
-    (acc, key) => {
-      if (fields.includes(key)) {
-        const [field, value] = adaptValues(key, data[key])
-        return { ...acc, [field]: value }
-      }
-      return acc
-    },
-    { _firestore_id: documentId }
-  )
-  return document
+
+  const document = sanitizeDocuments(fields, data)
+
+  return { _firestore_id: documentId, ...document }
 }
 
 /**

package/src/import/config.ts

@@ -48,6 +48,10 @@ program
     '-a, --api-key <api-key>',
     'The Meilisearch API key with permission to perform actions on indexes. Both the private key and the master key are valid choices but we strongly recommend using the private key for security purposes.'
   )
+  .option(
+    '-f, --fields-to-index <fields-to-index>',
+    'list of fields added in the document send to Meilisearch. Default contains all fields.'
+  )
 
 const validateInput = (
   value: string,
@@ -141,6 +145,13 @@ const questions = [
     name: 'apiKey',
     type: 'input',
   },
+  {
+    message:
+      ' What fields do you want to index in Meilisearch? Create a comma-separated list of the field names, or leave it blank to include all fields. The id field is always indexed even when omitted from the list.',
+    name: 'fieldsToIndex',
+    default: '*',
+    type: 'input',
+  },
 ]
 
 export interface CLIConfig {
@@ -180,6 +191,7 @@ export async function parseConfig(): Promise<CLIConfig> {
         indexUid: options.index,
         host: options.host,
         apiKey: options.apiKey,
+        fieldsToIndex: options.fieldsToIndex,
       },
     }
   }
@@ -191,6 +203,7 @@ export async function parseConfig(): Promise<CLIConfig> {
     batchSize,
     host,
     apiKey,
+    fieldsToIndex,
   } = await inquirer.prompt(questions)
 
   return {
@@ -202,6 +215,7 @@ export async function parseConfig(): Promise<CLIConfig> {
       indexUid: index,
       host: host,
       apiKey: apiKey,
+      fieldsToIndex: fieldsToIndex,
     },
   }
 }

package/src/import/index.ts

@@ -78,7 +78,11 @@ async function retrieveCollectionFromFirestore(
     const docs = snapshot.docs
 
     if (docs.length === 0) break
-    total += await sendDocumentsToMeilisearch(docs, index)
+    total += await sendDocumentsToMeilisearch(
+      docs,
+      index,
+      config.meilisearch?.fieldsToIndex || ''
+    )
 
     if (docs.length) {
       lastDocument = docs[docs.length - 1]
@@ -92,16 +96,17 @@ async function retrieveCollectionFromFirestore(
 
 /**
  * Adapts documents and indexes them in Meilisearch.
- * @param {any} docs
+ * @param {Change<DocumentSnapshot>} docs
  * @param {Index} index
- * @param {Change<DocumentSnapshot>} change
+ * @param {string} fieldsToIndex list of fields added in the document send to Meilisearch.
  */
 async function sendDocumentsToMeilisearch(
   docs: DocumentSnapshot[],
-  index: Index
+  index: Index,
+  fieldsToIndex: string
 ): Promise<number> {
   const document = docs.map(snapshot => {
-    return adaptDocument(snapshot.id, snapshot)
+    return adaptDocument(snapshot.id, snapshot, fieldsToIndex)
   })
   try {
     await index.addDocuments(document, { primaryKey: '_firestore_id' })

package/src/index.ts

@@ -66,7 +66,11 @@ async function handleAddDocument(
   try {
     logs.addDocument(documentId)
     if (validateDocumentId(documentId)) {
-      const document = adaptDocument(documentId, snapshot)
+      const document = adaptDocument(
+        documentId,
+        snapshot,
+        config.meilisearch.fieldsToIndex || ''
+      )
       const { taskUid } = await index.addDocuments([document], {
         primaryKey: '_firestore_id',
       })
@@ -119,7 +123,11 @@ async function handleUpdateDocument(
   try {
     logs.updateDocument(documentId)
     if (validateDocumentId(documentId)) {
-      const document = adaptDocument(documentId, after)
+      const document = adaptDocument(
+        documentId,
+        after,
+        config.meilisearch.fieldsToIndex || ''
+      )
       const { taskUid } = await index.addDocuments([document])
 
       logger.info(

package/src/util.ts

@@ -17,7 +17,6 @@
 
 import { DocumentSnapshot } from 'firebase-functions/lib/providers/firestore'
 import { Change } from 'firebase-functions'
-import { config } from './config'
 
 export enum ChangeType {
   CREATE,
@@ -53,11 +52,38 @@ export function getChangedDocumentId(change: Change<DocumentSnapshot>): string {
 }
 
 /**
- * Returns the MEILISEARCH_FIELDS_TO_INDEX value from the config file and formats it.
+ * Parse the fieldsToIndex string into an array.
+ *
+ * @param  {string} fieldsToIndex
  * @return {string[]} An array of fields.
  */
-export function getFieldsToIndex(): string[] {
-  return config.meilisearch.fieldsToIndex
-    ? config.meilisearch.fieldsToIndex.split(/[ ,]+/)
-    : []
+export function parseFieldsToIndex(fieldsToIndex: string): string[] {
+  return fieldsToIndex ? fieldsToIndex.split(/[ ,]+/) : []
+}
+
+/**
+ * Remove unwanted fields from the document before it is send to Meilisearch.
+ *
+ * @param  {string[]} fieldsToIndex
+ * @param  {Record<string, any>} document
+ * @return {Record<string, any>} sanitized document
+ *
+ */
+export function sanitizeDocuments(
+  fieldsToIndex: string[],
+  document: Record<string, any>
+): Record<string, any> {
+  if (fieldsToIndex.length === 0) {
+    return document
+  }
+  if (fieldsToIndex.includes('*')) {
+    return document
+  }
+
+  for (const key in document) {
+    if (!fieldsToIndex.includes(key)) {
+      delete document[key]
+    }
+  }
+  return document
 }

package/src/version.ts

@@ -1 +1 @@
-export const version = '0.1.9'
+export const version = '0.1.11'