firestore-meilisearch 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.editorconfig +20 -0
- package/.github/ISSUE_TEMPLATE/bug_report.md +26 -0
- package/.github/ISSUE_TEMPLATE/feature_request.md +18 -0
- package/.github/ISSUE_TEMPLATE/other.md +7 -0
- package/.github/dependatbot.yml +23 -0
- package/.github/release-draft-template.yml +33 -0
- package/.github/scripts/check-release.sh +42 -0
- package/.github/workflows/publish.yml +30 -0
- package/.github/workflows/release-drafter.yml +16 -0
- package/.github/workflows/test.yml +42 -0
- package/CHANGELOG.md +3 -0
- package/CONTRIBUTING.md +236 -0
- package/LICENSE +201 -0
- package/POSTINSTALL.md +40 -0
- package/PREINSTALL.md +42 -0
- package/README.md +128 -0
- package/bors.toml +8 -0
- package/dataset/firebase-export-metadata.json +8 -0
- package/dataset/firestore_export/all_namespaces/all_kinds/all_namespaces_all_kinds.export_metadata +0 -0
- package/dataset/firestore_export/all_namespaces/all_kinds/output-0 +0 -0
- package/dataset/firestore_export/firestore_export.overall_export_metadata +0 -0
- package/extension.yaml +176 -0
- package/firebase.json +20 -0
- package/functions/.eslintignore +2 -0
- package/functions/.eslintrc.js +54 -0
- package/functions/__tests__/__mocks__/console.ts +7 -0
- package/functions/__tests__/adapter.test.ts +98 -0
- package/functions/__tests__/config.test.ts +130 -0
- package/functions/__tests__/data/document.ts +11 -0
- package/functions/__tests__/data/environment.ts +9 -0
- package/functions/__tests__/functions.test.ts +280 -0
- package/functions/__tests__/jest.setup.ts +1 -0
- package/functions/__tests__/test.types.d.ts +5 -0
- package/functions/__tests__/tsconfig.json +5 -0
- package/functions/__tests__/util.test.ts +200 -0
- package/functions/jest.config.js +12 -0
- package/functions/lib/adapter.js +61 -0
- package/functions/lib/config.js +13 -0
- package/functions/lib/import/config.js +127 -0
- package/functions/lib/import/index.js +93 -0
- package/functions/lib/index.js +90 -0
- package/functions/lib/logs.js +97 -0
- package/functions/lib/meilisearch/create-index.js +17 -0
- package/functions/lib/meilisearch-index.js +17 -0
- package/functions/lib/types.js +2 -0
- package/functions/lib/util.js +47 -0
- package/functions/lib/version.js +4 -0
- package/functions/package.json +53 -0
- package/functions/src/adapter.ts +106 -0
- package/functions/src/config.ts +34 -0
- package/functions/src/import/config.ts +207 -0
- package/functions/src/import/index.ts +115 -0
- package/functions/src/index.ts +103 -0
- package/functions/src/logs.ts +107 -0
- package/functions/src/meilisearch/create-index.ts +20 -0
- package/functions/src/types.ts +8 -0
- package/functions/src/util.ts +63 -0
- package/functions/src/version.ts +1 -0
- package/functions/tsconfig.eslint.json +13 -0
- package/functions/tsconfig.json +23 -0
- package/functions/yarn.lock +5306 -0
- package/guides/IMPORT_EXISTING_DOCUMENTS.md +74 -0
- package/package.json +21 -0
- package/script/version.sh +51 -0
- package/test-params-example.env +9 -0
|
@@ -0,0 +1,200 @@
|
|
|
1
|
+
import * as firebaseFunctionsTestInit from 'firebase-functions-test'
|
|
2
|
+
import mockedEnv from 'mocked-env'
|
|
3
|
+
import { getChangeType, ChangeType, getChangedDocumentId } from '../src/util'
|
|
4
|
+
import defaultEnvironment from './data/environment'
|
|
5
|
+
|
|
6
|
+
describe('getChangeType', () => {
|
|
7
|
+
const firebaseMock = firebaseFunctionsTestInit()
|
|
8
|
+
|
|
9
|
+
test('return a create change type', () => {
|
|
10
|
+
const beforeSnapshot = firebaseMock.firestore.makeDocumentSnapshot(
|
|
11
|
+
{},
|
|
12
|
+
'docs/1'
|
|
13
|
+
)
|
|
14
|
+
const afterSnapshot = firebaseMock.firestore.makeDocumentSnapshot(
|
|
15
|
+
{ foo: 'bar' },
|
|
16
|
+
'docs/1'
|
|
17
|
+
)
|
|
18
|
+
|
|
19
|
+
const documentChange = firebaseMock.makeChange(
|
|
20
|
+
beforeSnapshot,
|
|
21
|
+
afterSnapshot
|
|
22
|
+
)
|
|
23
|
+
|
|
24
|
+
const changeType: ChangeType = getChangeType(documentChange)
|
|
25
|
+
|
|
26
|
+
expect(changeType).toEqual(ChangeType.CREATE)
|
|
27
|
+
})
|
|
28
|
+
|
|
29
|
+
test('return a update change type', () => {
|
|
30
|
+
const beforeSnapshot = firebaseMock.firestore.makeDocumentSnapshot(
|
|
31
|
+
{ foo: 'bar' },
|
|
32
|
+
'docs/1'
|
|
33
|
+
)
|
|
34
|
+
const afterSnapshot = firebaseMock.firestore.makeDocumentSnapshot(
|
|
35
|
+
{ foo: 'bars' },
|
|
36
|
+
'docs/1'
|
|
37
|
+
)
|
|
38
|
+
|
|
39
|
+
const documentChange = firebaseMock.makeChange(
|
|
40
|
+
beforeSnapshot,
|
|
41
|
+
afterSnapshot
|
|
42
|
+
)
|
|
43
|
+
|
|
44
|
+
const changeType: ChangeType = getChangeType(documentChange)
|
|
45
|
+
|
|
46
|
+
expect(changeType).toEqual(ChangeType.UPDATE)
|
|
47
|
+
})
|
|
48
|
+
|
|
49
|
+
test('return a delete change type', () => {
|
|
50
|
+
const beforeSnapshot = firebaseMock.firestore.makeDocumentSnapshot(
|
|
51
|
+
{ foo: 'bar' },
|
|
52
|
+
'docs/1'
|
|
53
|
+
)
|
|
54
|
+
const afterSnapshot = firebaseMock.firestore.makeDocumentSnapshot(
|
|
55
|
+
{},
|
|
56
|
+
'docs/1'
|
|
57
|
+
)
|
|
58
|
+
|
|
59
|
+
const documentChange = firebaseMock.makeChange(
|
|
60
|
+
beforeSnapshot,
|
|
61
|
+
afterSnapshot
|
|
62
|
+
)
|
|
63
|
+
|
|
64
|
+
const changeType: ChangeType = getChangeType(documentChange)
|
|
65
|
+
|
|
66
|
+
expect(changeType).toEqual(ChangeType.DELETE)
|
|
67
|
+
})
|
|
68
|
+
})
|
|
69
|
+
|
|
70
|
+
describe('getChangedDocumentId', () => {
|
|
71
|
+
const firebaseMock = firebaseFunctionsTestInit()
|
|
72
|
+
|
|
73
|
+
test('return id after create document', () => {
|
|
74
|
+
const beforeSnapshot = firebaseMock.firestore.makeDocumentSnapshot(
|
|
75
|
+
{},
|
|
76
|
+
'docs/1'
|
|
77
|
+
)
|
|
78
|
+
const afterSnapshot = firebaseMock.firestore.makeDocumentSnapshot(
|
|
79
|
+
{ foo: 'bar' },
|
|
80
|
+
'docs/2'
|
|
81
|
+
)
|
|
82
|
+
|
|
83
|
+
const documentChange = firebaseMock.makeChange(
|
|
84
|
+
beforeSnapshot,
|
|
85
|
+
afterSnapshot
|
|
86
|
+
)
|
|
87
|
+
|
|
88
|
+
const id: string = getChangedDocumentId(documentChange)
|
|
89
|
+
|
|
90
|
+
expect(id).toEqual('2')
|
|
91
|
+
})
|
|
92
|
+
|
|
93
|
+
test('return id after update document', () => {
|
|
94
|
+
const beforeSnapshot = firebaseMock.firestore.makeDocumentSnapshot(
|
|
95
|
+
{ foo: 'bar' },
|
|
96
|
+
'docs/1'
|
|
97
|
+
)
|
|
98
|
+
const afterSnapshot = firebaseMock.firestore.makeDocumentSnapshot(
|
|
99
|
+
{ foo: 'bars' },
|
|
100
|
+
'docs/2'
|
|
101
|
+
)
|
|
102
|
+
|
|
103
|
+
const documentChange = firebaseMock.makeChange(
|
|
104
|
+
beforeSnapshot,
|
|
105
|
+
afterSnapshot
|
|
106
|
+
)
|
|
107
|
+
|
|
108
|
+
const id: string = getChangedDocumentId(documentChange)
|
|
109
|
+
|
|
110
|
+
expect(id).toEqual('2')
|
|
111
|
+
})
|
|
112
|
+
|
|
113
|
+
test('return id after delete document', () => {
|
|
114
|
+
const beforeSnapshot = firebaseMock.firestore.makeDocumentSnapshot(
|
|
115
|
+
{ foo: 'bar' },
|
|
116
|
+
'docs/1'
|
|
117
|
+
)
|
|
118
|
+
const afterSnapshot = firebaseMock.firestore.makeDocumentSnapshot(
|
|
119
|
+
{},
|
|
120
|
+
'docs/2'
|
|
121
|
+
)
|
|
122
|
+
|
|
123
|
+
const documentChange = firebaseMock.makeChange(
|
|
124
|
+
beforeSnapshot,
|
|
125
|
+
afterSnapshot
|
|
126
|
+
)
|
|
127
|
+
|
|
128
|
+
const id: string = getChangedDocumentId(documentChange)
|
|
129
|
+
|
|
130
|
+
expect(id).toEqual('1')
|
|
131
|
+
})
|
|
132
|
+
})
|
|
133
|
+
|
|
134
|
+
describe('getFieldsToIndex', () => {
|
|
135
|
+
let util
|
|
136
|
+
let restoreEnv
|
|
137
|
+
let mockGetFieldsToIndex
|
|
138
|
+
const config = global.config
|
|
139
|
+
|
|
140
|
+
beforeEach(() => {
|
|
141
|
+
jest.resetModules()
|
|
142
|
+
restoreEnv = mockedEnv(defaultEnvironment)
|
|
143
|
+
})
|
|
144
|
+
afterEach(() => restoreEnv())
|
|
145
|
+
|
|
146
|
+
test('configuration detected from environment variables', () => {
|
|
147
|
+
const mockConfig = config()
|
|
148
|
+
expect(mockConfig).toMatchSnapshot()
|
|
149
|
+
})
|
|
150
|
+
|
|
151
|
+
test('return empty list', () => {
|
|
152
|
+
util = require('../src/util')
|
|
153
|
+
mockGetFieldsToIndex = util.getFieldsToIndex()
|
|
154
|
+
expect(mockGetFieldsToIndex).toMatchObject([])
|
|
155
|
+
})
|
|
156
|
+
|
|
157
|
+
test('return list with one field', () => {
|
|
158
|
+
restoreEnv = mockedEnv({
|
|
159
|
+
...defaultEnvironment,
|
|
160
|
+
MEILISEARCH_FIELDS_TO_INDEX: 'field',
|
|
161
|
+
})
|
|
162
|
+
util = require('../src/util')
|
|
163
|
+
mockGetFieldsToIndex = util.getFieldsToIndex()
|
|
164
|
+
expect(mockGetFieldsToIndex).toMatchObject(['field'])
|
|
165
|
+
})
|
|
166
|
+
|
|
167
|
+
test('return list with multiple fields', () => {
|
|
168
|
+
restoreEnv = mockedEnv({
|
|
169
|
+
...defaultEnvironment,
|
|
170
|
+
MEILISEARCH_FIELDS_TO_INDEX: 'field1,field2,field3',
|
|
171
|
+
})
|
|
172
|
+
util = require('../src/util')
|
|
173
|
+
mockGetFieldsToIndex = util.getFieldsToIndex()
|
|
174
|
+
expect(mockGetFieldsToIndex).toMatchObject(['field1', 'field2', 'field3'])
|
|
175
|
+
})
|
|
176
|
+
|
|
177
|
+
test('return list with multiple fields and spaces', () => {
|
|
178
|
+
restoreEnv = mockedEnv({
|
|
179
|
+
...defaultEnvironment,
|
|
180
|
+
MEILISEARCH_FIELDS_TO_INDEX: 'field1, field2, field3',
|
|
181
|
+
})
|
|
182
|
+
util = require('../src/util')
|
|
183
|
+
mockGetFieldsToIndex = util.getFieldsToIndex()
|
|
184
|
+
expect(mockGetFieldsToIndex).toMatchObject(['field1', 'field2', 'field3'])
|
|
185
|
+
})
|
|
186
|
+
|
|
187
|
+
test('return list of fiels with underscore', () => {
|
|
188
|
+
restoreEnv = mockedEnv({
|
|
189
|
+
...defaultEnvironment,
|
|
190
|
+
MEILISEARCH_FIELDS_TO_INDEX: 'field_1,field_2,field_3',
|
|
191
|
+
})
|
|
192
|
+
util = require('../src/util')
|
|
193
|
+
mockGetFieldsToIndex = util.getFieldsToIndex()
|
|
194
|
+
expect(mockGetFieldsToIndex).toMatchObject([
|
|
195
|
+
'field_1',
|
|
196
|
+
'field_2',
|
|
197
|
+
'field_3',
|
|
198
|
+
])
|
|
199
|
+
})
|
|
200
|
+
})
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
module.exports = {
|
|
2
|
+
rootDir: './',
|
|
3
|
+
preset: 'ts-jest',
|
|
4
|
+
globals: {
|
|
5
|
+
'ts-jest': {
|
|
6
|
+
tsconfig: '<rootDir>/__tests__/tsconfig.json',
|
|
7
|
+
},
|
|
8
|
+
},
|
|
9
|
+
testEnvironment: 'node',
|
|
10
|
+
testMatch: ['**/__tests__/*.test.ts'],
|
|
11
|
+
setupFiles: ['<rootDir>/__tests__/jest.setup.ts'],
|
|
12
|
+
}
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.adaptValues = exports.adaptDocument = void 0;
|
|
4
|
+
const firestore_1 = require("firebase-admin/lib/firestore");
|
|
5
|
+
const util_1 = require("./util");
|
|
6
|
+
const logs = require("./logs");
|
|
7
|
+
/**
|
|
8
|
+
* Adapts documents from the Firestore database to Meilisearch compatible documents.
|
|
9
|
+
* @param {string} documentId Document id.
|
|
10
|
+
* @param {DocumentSnapshot} snapshot Snapshot of the data contained in the document read from your Firestore database.
|
|
11
|
+
* @return {Record<string, any>} A properly formatted document to be added or updated in Meilisearch.
|
|
12
|
+
*/
|
|
13
|
+
function adaptDocument(documentId, snapshot) {
|
|
14
|
+
const fields = (0, util_1.getFieldsToIndex)();
|
|
15
|
+
const data = snapshot.data() || {};
|
|
16
|
+
if ('_firestore_id' in data) {
|
|
17
|
+
delete data.id;
|
|
18
|
+
}
|
|
19
|
+
if (fields.length === 0) {
|
|
20
|
+
return { _firestore_id: documentId, ...data };
|
|
21
|
+
}
|
|
22
|
+
const document = Object.keys(data).reduce((acc, key) => {
|
|
23
|
+
if (fields.includes(key)) {
|
|
24
|
+
const [field, value] = adaptValues(key, data[key]);
|
|
25
|
+
return { ...acc, [field]: value };
|
|
26
|
+
}
|
|
27
|
+
return acc;
|
|
28
|
+
}, { _firestore_id: documentId });
|
|
29
|
+
return document;
|
|
30
|
+
}
|
|
31
|
+
exports.adaptDocument = adaptDocument;
|
|
32
|
+
/**
|
|
33
|
+
* Checks and adapts each values to be compatible with Meilisearch documents.
|
|
34
|
+
* @param {string} field
|
|
35
|
+
* @param {FirestoreRow} value
|
|
36
|
+
* @return {[string,FirestoreRow]} A properly formatted array of field and value.
|
|
37
|
+
*/
|
|
38
|
+
function adaptValues(field, value) {
|
|
39
|
+
if (value instanceof firestore_1.GeoPoint) {
|
|
40
|
+
if (field === '_geo') {
|
|
41
|
+
logs.infoGeoPoint(true);
|
|
42
|
+
return [field, adaptGeoPoint(value)];
|
|
43
|
+
}
|
|
44
|
+
else {
|
|
45
|
+
logs.infoGeoPoint(false);
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
return [field, value];
|
|
49
|
+
}
|
|
50
|
+
exports.adaptValues = adaptValues;
|
|
51
|
+
/**
|
|
52
|
+
* Adapts GeoPoint Firestore instance to fit with Meilisearch geo point.
|
|
53
|
+
* @param {GeoPoint} geoPoint GeoPoint Firestore object.
|
|
54
|
+
* @return {MeilisearchGeoPoint} A properly formatted geo point for Meilisearch.
|
|
55
|
+
*/
|
|
56
|
+
const adaptGeoPoint = (geoPoint) => {
|
|
57
|
+
return {
|
|
58
|
+
lat: geoPoint.latitude,
|
|
59
|
+
lng: geoPoint.longitude,
|
|
60
|
+
};
|
|
61
|
+
};
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.config = void 0;
|
|
4
|
+
exports.config = {
|
|
5
|
+
location: process.env.LOCATION || 'europe-west1',
|
|
6
|
+
collectionPath: process.env.COLLECTION_PATH || '',
|
|
7
|
+
meilisearch: {
|
|
8
|
+
host: process.env.MEILISEARCH_HOST || '',
|
|
9
|
+
apiKey: process.env.MEILISEARCH_API_KEY || '',
|
|
10
|
+
indexUid: process.env.MEILISEARCH_INDEX_NAME || '',
|
|
11
|
+
fieldsToIndex: process.env.MEILISEARCH_FIELDS_TO_INDEX || '',
|
|
12
|
+
},
|
|
13
|
+
};
|
|
@@ -0,0 +1,127 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.parseConfig = void 0;
|
|
4
|
+
const commander_1 = require("commander");
|
|
5
|
+
const inquirer = require("inquirer");
|
|
6
|
+
const FIRESTORE_VALID_CHARACTERS = /^[^/]+$/;
|
|
7
|
+
const FIRESTORE_COLLECTION_NAME_MAX_CHARS = 6144;
|
|
8
|
+
const PROJECT_ID_MAX_CHARS = 30;
|
|
9
|
+
const MEILISEARCH_VALID_CHARACTERS = /^[a-zA-Z-_0-9,]*$/;
|
|
10
|
+
const MEILISEARCH_UID_MAX_CHARS = 6144;
|
|
11
|
+
// Create a new interface to manage the arguments from the non-interactive command line.
|
|
12
|
+
const program = new commander_1.Command();
|
|
13
|
+
program
|
|
14
|
+
.name('firestore-meilisearch')
|
|
15
|
+
.option('--non-interactive', 'Parse all input from command line flags instead of prompting the caller.', false)
|
|
16
|
+
.option('-P, --project <project>', 'Firebase Project ID for project containing the Cloud Firestore database.')
|
|
17
|
+
.option('-s, --source-collection-path <source-collection-path>', 'The path of the Cloud Firestore Collection to import from. (This may, or may not, be the same Collection for which you plan to mirror changes.)')
|
|
18
|
+
.option('-q, --query-collection-group [true|false]', "Use 'true' for a collection group query, otherwise a collection query is performed.")
|
|
19
|
+
.option('-i, --index <index>', "The Uid of the index in Meilisearch to import to. (An index will be created if it doesn't already exist.)")
|
|
20
|
+
.option('-b, --batch-size [batch-size]', 'Number of documents to stream into Meilisearch at once.', value => parseInt(value, 10), 1000)
|
|
21
|
+
.option('-H, --host <host>', 'The Host of your Meilisearch database. Example: http://localhost:7700.')
|
|
22
|
+
.option('-a, --api-key <api-key>', 'The Meilisearch API key with permission to perform actions on indexes. Both the private key and the master key are valid choices but we strongly recommend using the private key for security purposes.');
|
|
23
|
+
const validateInput = (value, name, regex, sizeLimit) => {
|
|
24
|
+
if (!value || typeof value !== 'string' || value.trim() === '') {
|
|
25
|
+
return `Please supply a ${name}`;
|
|
26
|
+
}
|
|
27
|
+
if (value.trim().length > sizeLimit) {
|
|
28
|
+
return `${name} must be at most ${sizeLimit} characters long`;
|
|
29
|
+
}
|
|
30
|
+
if (!value.match(regex)) {
|
|
31
|
+
return `The ${name} does not match the regular expression provided`;
|
|
32
|
+
}
|
|
33
|
+
return true;
|
|
34
|
+
};
|
|
35
|
+
const validateBatchSize = (value) => {
|
|
36
|
+
if (/^\d+$/.test(value) == false) {
|
|
37
|
+
return `The batchsize ${value} should be a number`;
|
|
38
|
+
}
|
|
39
|
+
return parseInt(value, 10) > 0;
|
|
40
|
+
};
|
|
41
|
+
// Questions for the interactive user interface in command line.
|
|
42
|
+
const questions = [
|
|
43
|
+
{
|
|
44
|
+
message: 'What is your Firebase project ID?',
|
|
45
|
+
name: 'project',
|
|
46
|
+
type: 'input',
|
|
47
|
+
validate: value => validateInput(value, 'project ID', FIRESTORE_VALID_CHARACTERS, PROJECT_ID_MAX_CHARS),
|
|
48
|
+
},
|
|
49
|
+
{
|
|
50
|
+
message: 'What is the path of the Cloud Firestore Collection you would like to import from? ' +
|
|
51
|
+
'(This may, or may not, be the same Collection for which you plan to mirror changes.)',
|
|
52
|
+
name: 'sourceCollectionPath',
|
|
53
|
+
type: 'input',
|
|
54
|
+
validate: value => validateInput(value, 'collection path', FIRESTORE_VALID_CHARACTERS, FIRESTORE_COLLECTION_NAME_MAX_CHARS),
|
|
55
|
+
},
|
|
56
|
+
{
|
|
57
|
+
message: 'Would you like to import documents via a Collection Group query?',
|
|
58
|
+
name: 'queryCollectionGroup',
|
|
59
|
+
type: 'confirm',
|
|
60
|
+
default: false,
|
|
61
|
+
},
|
|
62
|
+
{
|
|
63
|
+
message: "What is the Uid of the Meilisearch index that you would like to use? (A index will be created if it doesn't already exist)",
|
|
64
|
+
name: 'index',
|
|
65
|
+
type: 'input',
|
|
66
|
+
validate: value => validateInput(value, 'index', MEILISEARCH_VALID_CHARACTERS, MEILISEARCH_UID_MAX_CHARS),
|
|
67
|
+
},
|
|
68
|
+
{
|
|
69
|
+
message: 'How many documents should the import stream into Meilisearch at once?',
|
|
70
|
+
name: 'batchSize',
|
|
71
|
+
type: 'input',
|
|
72
|
+
default: 1000,
|
|
73
|
+
validate: validateBatchSize,
|
|
74
|
+
},
|
|
75
|
+
{
|
|
76
|
+
message: 'What is the host of the Meilisearch database that you would like to use? Example: http://localhost:7700.',
|
|
77
|
+
name: 'host',
|
|
78
|
+
type: 'input',
|
|
79
|
+
},
|
|
80
|
+
{
|
|
81
|
+
message: 'Which Meilisearch API key with permission to perform actions on indexes do you like to use? Both the private key and the master key are valid choices but we strongly recommend using the private key for security purposes.',
|
|
82
|
+
name: 'apiKey',
|
|
83
|
+
type: 'input',
|
|
84
|
+
},
|
|
85
|
+
];
|
|
86
|
+
/**
|
|
87
|
+
* Parse the argument from the interactive or non-interactive command line.
|
|
88
|
+
*/
|
|
89
|
+
async function parseConfig() {
|
|
90
|
+
program.parse(process.argv);
|
|
91
|
+
const options = program.opts();
|
|
92
|
+
if (options.nonInteractive) {
|
|
93
|
+
if (options.project === undefined ||
|
|
94
|
+
options.sourceCollectionPath === undefined ||
|
|
95
|
+
options.index === undefined ||
|
|
96
|
+
options.host === undefined ||
|
|
97
|
+
options.apiKey === undefined ||
|
|
98
|
+
!validateBatchSize(options.batchSize)) {
|
|
99
|
+
program.outputHelp();
|
|
100
|
+
process.exit(1);
|
|
101
|
+
}
|
|
102
|
+
return {
|
|
103
|
+
projectId: options.project,
|
|
104
|
+
sourceCollectionPath: options.sourceCollectionPath,
|
|
105
|
+
queryCollectionGroup: options.queryCollectionGroup === 'true',
|
|
106
|
+
batchSize: options.batchSize,
|
|
107
|
+
meilisearch: {
|
|
108
|
+
indexUid: options.index,
|
|
109
|
+
host: options.host,
|
|
110
|
+
apiKey: options.apiKey,
|
|
111
|
+
},
|
|
112
|
+
};
|
|
113
|
+
}
|
|
114
|
+
const { project, sourceCollectionPath, queryCollectionGroup, index, batchSize, host, apiKey, } = await inquirer.prompt(questions);
|
|
115
|
+
return {
|
|
116
|
+
projectId: project,
|
|
117
|
+
sourceCollectionPath: sourceCollectionPath,
|
|
118
|
+
queryCollectionGroup: queryCollectionGroup,
|
|
119
|
+
batchSize: batchSize,
|
|
120
|
+
meilisearch: {
|
|
121
|
+
indexUid: index,
|
|
122
|
+
host: host,
|
|
123
|
+
apiKey: apiKey,
|
|
124
|
+
},
|
|
125
|
+
};
|
|
126
|
+
}
|
|
127
|
+
exports.parseConfig = parseConfig;
|
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
"use strict";
|
|
3
|
+
/*
|
|
4
|
+
* Copyright 2022 Meilisearch
|
|
5
|
+
*
|
|
6
|
+
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
7
|
+
* you may not use this file except in compliance with the License.
|
|
8
|
+
* You may obtain a copy of the License at
|
|
9
|
+
*
|
|
10
|
+
* https://www.apache.org/licenses/LICENSE-2.0
|
|
11
|
+
*
|
|
12
|
+
* Unless required by applicable law or agreed to in writing, software
|
|
13
|
+
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
14
|
+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
15
|
+
* See the License for the specific language governing permissions and
|
|
16
|
+
* limitations under the License.
|
|
17
|
+
*/
|
|
18
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
19
|
+
const admin = require("firebase-admin");
|
|
20
|
+
const config_1 = require("./config");
|
|
21
|
+
const logs = require("../logs");
|
|
22
|
+
const adapter_1 = require("../adapter");
|
|
23
|
+
const create_index_1 = require("../meilisearch/create-index");
|
|
24
|
+
const run = async () => {
|
|
25
|
+
// Retrieve all arguments from the commande line.
|
|
26
|
+
const config = await (0, config_1.parseConfig)();
|
|
27
|
+
// Initialize Firebase using the Google Credentials in the GOOGLE_APPLICATION_CREDENTIALS environment variable.
|
|
28
|
+
admin.initializeApp({
|
|
29
|
+
credential: admin.credential.applicationDefault(),
|
|
30
|
+
databaseURL: `https://${config.projectId}.firebaseio.com`,
|
|
31
|
+
});
|
|
32
|
+
const database = admin.firestore();
|
|
33
|
+
// Initialize Meilisearch index.
|
|
34
|
+
const index = (0, create_index_1.initMeilisearchIndex)(config.meilisearch);
|
|
35
|
+
await retrieveCollectionFromFirestore(database, config, index);
|
|
36
|
+
};
|
|
37
|
+
/**
|
|
38
|
+
* Retrieves a collection or collection group in Firestore and aggregates the data.
|
|
39
|
+
* @param {FirebaseFirestore.Firestore} database
|
|
40
|
+
* @param {CLIConfig} config
|
|
41
|
+
* @param {Index} index
|
|
42
|
+
*/
|
|
43
|
+
async function retrieveCollectionFromFirestore(database, config, index) {
|
|
44
|
+
const batch = parseInt(config.batchSize);
|
|
45
|
+
let query;
|
|
46
|
+
let total = 0;
|
|
47
|
+
let batches = 0;
|
|
48
|
+
let lastDocument = null;
|
|
49
|
+
let lastBatchSize = batch;
|
|
50
|
+
while (lastBatchSize === batch) {
|
|
51
|
+
batches++;
|
|
52
|
+
if (config.queryCollectionGroup) {
|
|
53
|
+
query = database.collectionGroup(config.sourceCollectionPath).limit(batch);
|
|
54
|
+
}
|
|
55
|
+
else {
|
|
56
|
+
query = database.collection(config.sourceCollectionPath).limit(batch);
|
|
57
|
+
}
|
|
58
|
+
if (lastDocument !== null) {
|
|
59
|
+
query = query.startAfter(lastDocument);
|
|
60
|
+
}
|
|
61
|
+
const snapshot = await query.get();
|
|
62
|
+
const docs = snapshot.docs;
|
|
63
|
+
if (docs.length === 0)
|
|
64
|
+
break;
|
|
65
|
+
total += await sendDocumentsToMeilisearch(docs, index);
|
|
66
|
+
if (docs.length) {
|
|
67
|
+
lastDocument = docs[docs.length - 1];
|
|
68
|
+
}
|
|
69
|
+
lastBatchSize = docs.length;
|
|
70
|
+
}
|
|
71
|
+
logs.importData(total, batches);
|
|
72
|
+
return total;
|
|
73
|
+
}
|
|
74
|
+
/**
|
|
75
|
+
* Adapts documents and indexes them in Meilisearch.
|
|
76
|
+
* @param {any} docs
|
|
77
|
+
* @param {Index} index
|
|
78
|
+
* @param {Change<DocumentSnapshot>} change
|
|
79
|
+
*/
|
|
80
|
+
async function sendDocumentsToMeilisearch(docs, index) {
|
|
81
|
+
const document = docs.map(snapshot => {
|
|
82
|
+
return (0, adapter_1.adaptDocument)(snapshot.id, snapshot);
|
|
83
|
+
});
|
|
84
|
+
try {
|
|
85
|
+
await index.addDocuments(document, { primaryKey: '_firestore_id' });
|
|
86
|
+
}
|
|
87
|
+
catch (e) {
|
|
88
|
+
logs.error(e);
|
|
89
|
+
return 0;
|
|
90
|
+
}
|
|
91
|
+
return document.length;
|
|
92
|
+
}
|
|
93
|
+
void run();
|
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.indexingWorker = void 0;
|
|
4
|
+
/*
|
|
5
|
+
* Copyright 2022 Meilisearch
|
|
6
|
+
*
|
|
7
|
+
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
8
|
+
* you may not use this file except in compliance with the License.
|
|
9
|
+
* You may obtain a copy of the License at
|
|
10
|
+
*
|
|
11
|
+
* https://www.apache.org/licenses/LICENSE-2.0
|
|
12
|
+
*
|
|
13
|
+
* Unless required by applicable law or agreed to in writing, software
|
|
14
|
+
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
15
|
+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
16
|
+
* See the License for the specific language governing permissions and
|
|
17
|
+
* limitations under the License.
|
|
18
|
+
*/
|
|
19
|
+
const functions = require("firebase-functions");
|
|
20
|
+
const create_index_1 = require("./meilisearch/create-index");
|
|
21
|
+
const util_1 = require("./util");
|
|
22
|
+
const logs = require("./logs");
|
|
23
|
+
const adapter_1 = require("./adapter");
|
|
24
|
+
const config_1 = require("./config");
|
|
25
|
+
const index = (0, create_index_1.initMeilisearchIndex)(config_1.config.meilisearch);
|
|
26
|
+
logs.init();
|
|
27
|
+
/**
|
|
28
|
+
* IndexingWorker is responsible for aggregating a defined field from a Firestore collection into a Meilisearch index.
|
|
29
|
+
* It is controlled by a Firestore handler.
|
|
30
|
+
*/
|
|
31
|
+
exports.indexingWorker = functions.handler.firestore.document.onWrite(async (change) => {
|
|
32
|
+
logs.start();
|
|
33
|
+
const changeType = (0, util_1.getChangeType)(change);
|
|
34
|
+
const documentId = (0, util_1.getChangedDocumentId)(change);
|
|
35
|
+
switch (changeType) {
|
|
36
|
+
case util_1.ChangeType.CREATE:
|
|
37
|
+
await handleAddDocument(documentId, change.after);
|
|
38
|
+
break;
|
|
39
|
+
case util_1.ChangeType.DELETE:
|
|
40
|
+
await handleDeleteDocument(documentId);
|
|
41
|
+
break;
|
|
42
|
+
case util_1.ChangeType.UPDATE:
|
|
43
|
+
await handleUpdateDocument(documentId, change.after);
|
|
44
|
+
break;
|
|
45
|
+
}
|
|
46
|
+
logs.complete();
|
|
47
|
+
});
|
|
48
|
+
/**
|
|
49
|
+
* Handle addition of a document in the Meilisearch index.
|
|
50
|
+
* @param {string} documentId Document id to add.
|
|
51
|
+
* @param {Change} snapshot Snapshot of the data contained in the document read from your Firestore database.
|
|
52
|
+
*/
|
|
53
|
+
async function handleAddDocument(documentId, snapshot) {
|
|
54
|
+
try {
|
|
55
|
+
const document = (0, adapter_1.adaptDocument)(documentId, snapshot);
|
|
56
|
+
await index.addDocuments([document], { primaryKey: '_firestore_id' });
|
|
57
|
+
logs.addDocument(documentId, document);
|
|
58
|
+
}
|
|
59
|
+
catch (e) {
|
|
60
|
+
logs.error(e);
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
/**
|
|
64
|
+
* Handle deletion of a document in the Meilisearch index.
|
|
65
|
+
* @param {string} documentId Document id to delete.
|
|
66
|
+
*/
|
|
67
|
+
async function handleDeleteDocument(documentId) {
|
|
68
|
+
try {
|
|
69
|
+
await index.deleteDocument(documentId);
|
|
70
|
+
logs.deleteDocument(documentId);
|
|
71
|
+
}
|
|
72
|
+
catch (e) {
|
|
73
|
+
logs.error(e);
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
/**
|
|
77
|
+
* Handle update of a document in the Meilisearch index.
|
|
78
|
+
* @param {string} documentId Document id to update.
|
|
79
|
+
* @param {Change} after Snapshot of the data contained in the document read from your Firestore database.
|
|
80
|
+
*/
|
|
81
|
+
async function handleUpdateDocument(documentId, after) {
|
|
82
|
+
try {
|
|
83
|
+
const document = (0, adapter_1.adaptDocument)(documentId, after);
|
|
84
|
+
await index.updateDocuments([document]);
|
|
85
|
+
logs.updateDocument(documentId, document);
|
|
86
|
+
}
|
|
87
|
+
catch (e) {
|
|
88
|
+
logs.error(e);
|
|
89
|
+
}
|
|
90
|
+
}
|
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.importData = exports.infoGeoPoint = exports.deleteDocument = exports.updateDocument = exports.addDocument = exports.complete = exports.error = exports.start = exports.init = void 0;
|
|
4
|
+
/*
|
|
5
|
+
* Copyright 2022 Meilisearch
|
|
6
|
+
*
|
|
7
|
+
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
8
|
+
* you may not use this file except in compliance with the License.
|
|
9
|
+
* You may obtain a copy of the License at
|
|
10
|
+
*
|
|
11
|
+
* https://www.apache.org/licenses/LICENSE-2.0
|
|
12
|
+
*
|
|
13
|
+
* Unless required by applicable law or agreed to in writing, software
|
|
14
|
+
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
15
|
+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
16
|
+
* See the License for the specific language governing permissions and
|
|
17
|
+
* limitations under the License.
|
|
18
|
+
*/
|
|
19
|
+
const firebase_functions_1 = require("firebase-functions");
|
|
20
|
+
const config_1 = require("./config");
|
|
21
|
+
/**
|
|
22
|
+
* Initialization logger.
|
|
23
|
+
*/
|
|
24
|
+
function init() {
|
|
25
|
+
firebase_functions_1.logger.log('Initializing extension with configuration', config_1.config);
|
|
26
|
+
}
|
|
27
|
+
exports.init = init;
|
|
28
|
+
/**
|
|
29
|
+
* Start logger.
|
|
30
|
+
*/
|
|
31
|
+
function start() {
|
|
32
|
+
firebase_functions_1.logger.log('Started execution of extension with configuration', config_1.config);
|
|
33
|
+
}
|
|
34
|
+
exports.start = start;
|
|
35
|
+
/**
|
|
36
|
+
* Error logger.
|
|
37
|
+
* @param {Error} err
|
|
38
|
+
*/
|
|
39
|
+
function error(err) {
|
|
40
|
+
firebase_functions_1.logger.error('Unhandled error occurred during processing:', err);
|
|
41
|
+
}
|
|
42
|
+
exports.error = error;
|
|
43
|
+
/**
|
|
44
|
+
* Complete logger.
|
|
45
|
+
*/
|
|
46
|
+
function complete() {
|
|
47
|
+
firebase_functions_1.logger.log('Completed execution of extension');
|
|
48
|
+
}
|
|
49
|
+
exports.complete = complete;
|
|
50
|
+
/**
|
|
51
|
+
* Log an addition of a document.
|
|
52
|
+
* @param {string} id Document id added.
|
|
53
|
+
* @param {object} data Data contained in the document.
|
|
54
|
+
*/
|
|
55
|
+
function addDocument(id, data) {
|
|
56
|
+
firebase_functions_1.logger.info(`Creating new document ${id} in Meilisearch index ${config_1.config.meilisearch.indexUid}`, data);
|
|
57
|
+
}
|
|
58
|
+
exports.addDocument = addDocument;
|
|
59
|
+
/**
|
|
60
|
+
* Log an update of a document.
|
|
61
|
+
* @param {string} id Document id updated.
|
|
62
|
+
* @param {object} data Data contained in the document.
|
|
63
|
+
*/
|
|
64
|
+
function updateDocument(id, data) {
|
|
65
|
+
firebase_functions_1.logger.info(`Updating document ${id} in Meilisearch index ${config_1.config.meilisearch.indexUid}`, data);
|
|
66
|
+
}
|
|
67
|
+
exports.updateDocument = updateDocument;
|
|
68
|
+
/**
|
|
69
|
+
* Log a deletion of a document.
|
|
70
|
+
* @param {string} id Document id deleted.
|
|
71
|
+
*/
|
|
72
|
+
function deleteDocument(id) {
|
|
73
|
+
firebase_functions_1.logger.info(`Deleting document ${id} in Meilisearch index ${config_1.config.meilisearch.indexUid}`);
|
|
74
|
+
}
|
|
75
|
+
exports.deleteDocument = deleteDocument;
|
|
76
|
+
/**
|
|
77
|
+
* Log a modification of geoPoint based on whether or not it has the correct naming to enable `geosearch` in Meilisearch.
|
|
78
|
+
* @param {boolean} hasGeoField a boolean value that indicates whether the field is correctly named to enable `geosearch` in Meilisearch.
|
|
79
|
+
*/
|
|
80
|
+
function infoGeoPoint(hasGeoField) {
|
|
81
|
+
if (hasGeoField) {
|
|
82
|
+
firebase_functions_1.logger.info(`A GeoPoint was found with the field name '_geo' for compatibility with Meilisearch the field 'latitude' was renamed to 'lat' and the field 'longitude' to 'lng'`);
|
|
83
|
+
}
|
|
84
|
+
else {
|
|
85
|
+
firebase_functions_1.logger.info(`A GeoPoint was found without the field name '_geo' if you want to use the geoSearch with Meilisearch rename it to '_geo'`);
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
exports.infoGeoPoint = infoGeoPoint;
|
|
89
|
+
/**
|
|
90
|
+
* Importation data logger.
|
|
91
|
+
* @param {number} total
|
|
92
|
+
* @param {number} batches
|
|
93
|
+
*/
|
|
94
|
+
function importData(total, batches) {
|
|
95
|
+
firebase_functions_1.logger.info(`Imported ${total} documents in ${batches} batches.`);
|
|
96
|
+
}
|
|
97
|
+
exports.importData = importData;
|