@servicetitan/acquisition-functions 0.1.0 → 0.3.0

package/README.md

@@ -14,6 +14,7 @@ const { configureProcessor } = require('@servicetitan/acquisition-functions');
 const { filterFetcher } = configureProcessor({
     workers: 10,
     mongoDb: mongoDbConnectionToDb,
+    fetchProperties: (zip) => mongoDbConnectionToDb.collection('properties').find({zipcode: zip}, {_id: 0}),
 });
 
 exports = async () => {
@@ -33,9 +34,5 @@ npm run test
 In Docker:
 
 ```sh
-# In root FE directory
-lerna run build --scope=@servicetitan/acquisition-functions
-
-# In package directory
-docker-compose up
+docker-compose up --from-exit-code=tests
 ```

package/dist/__tests__/assessorlastsaledate-converter.test.js

@@ -6,6 +6,7 @@ const setup_1 = require("./setup");
 describe('property-assessorlastsaledate-converter', () => {
     let processor;
     let base;
+    let metadata;
     (0, globals_1.beforeAll)(async () => {
         const client = await (0, setup_1.openConnection)();
         base = client.db('test');
@@ -18,7 +19,7 @@ describe('property-assessorlastsaledate-converter', () => {
                 mongoDb: base,
             });
             const { assessorLastSaleDateProcessor } = processor;
-            await assessorLastSaleDateProcessor.start('90001');
+            metadata = await assessorLastSaleDateProcessor.start('90001');
         });
         (0, globals_1.test)('should convert existing field to date', async () => {
             const result = await new Promise(resolve => {
@@ -46,6 +47,11 @@ describe('property-assessorlastsaledate-converter', () => {
             });
             (0, globals_1.expect)(result[0].assessorlastsaledate).toBe(null);
         });
+        (0, globals_1.test)('should return metadata', async () => {
+            (0, globals_1.expect)(metadata.errors).toBe(0);
+            (0, globals_1.expect)(metadata.properties).toBe(1);
+            (0, globals_1.expect)(metadata.requests).toBe(2);
+        });
     });
     describe('aggregate', () => {
         (0, globals_1.beforeEach)(async () => {

package/dist/__tests__/filter-fetcher.test.js

@@ -4,19 +4,24 @@ const globals_1 = require("@jest/globals");
 const index_1 = require("../index");
 const setup_1 = require("./setup");
 describe('property-assessorlastsaledate-converter', () => {
-    let processor;
     let base;
+    let metadata;
     (0, globals_1.beforeAll)(async () => {
         const client = await (0, setup_1.openConnection)();
         base = client.db('test');
         await (0, setup_1.setupProperties)(base);
-        processor = (0, index_1.configureProcessor)({
+        const { filterFetcher } = (0, index_1.configureProcessor)({
             workers: 10,
             mongoDb: base,
+            fetchProperties: (0, setup_1.getFetchPropertiesForFilters)(base),
         });
-        const { filterFetcher, propertyUseGroupProcessor } = processor;
-        await propertyUseGroupProcessor.start('90001');
-        await filterFetcher.start('90001-90002');
+        const { propertyUseGroupProcessor } = (0, index_1.configureProcessor)({
+            workers: 10,
+            mongoDb: base,
+            fetchProperties: (0, setup_1.getFetchPropertiesForFix)(base),
+        });
+        await propertyUseGroupProcessor.start(['90001', '90002']);
+        metadata = await filterFetcher.start(['90001', '90002']);
     });
     (0, globals_1.test)('should have empty filter for missing zip', async () => {
         const result = await base.collection('filters').findOne({
@@ -30,6 +35,12 @@ describe('property-assessorlastsaledate-converter', () => {
         (0, globals_1.expect)(result?.propertyusestandardized.length).toBe(0);
         (0, globals_1.expect)(result?.utilitieswatersource.length).toBe(0);
     });
+    (0, globals_1.test)('should return metadata', async () => {
+        (0, globals_1.expect)(metadata.errors).toBe(0);
+        (0, globals_1.expect)(metadata.properties).toBe(2);
+        (0, globals_1.expect)(metadata.requests).toBe(6);
+        (0, globals_1.expect)(metadata.filters).toBe(2);
+    });
     (0, globals_1.test)('should form correct filter', async () => {
         const result = await base.collection('filters').findOne({
             zipcode: '90001',

package/dist/__tests__/property-fix-use-group.test.js

@@ -11,14 +11,21 @@ describe('property-use-group-converter', () => {
     });
     describe('processor', () => {
         let processor;
+        let metadata;
         (0, globals_1.beforeAll)(async () => {
             await (0, setup_1.setupProperties)(base);
             processor = (0, index_1.configureProcessor)({
                 workers: 1,
                 mongoDb: base,
+                fetchProperties: (0, setup_1.getFetchPropertiesForFix)(base),
             });
             const { propertyUseGroupProcessor } = processor;
-            await propertyUseGroupProcessor.start('90001-90002');
+            metadata = await propertyUseGroupProcessor.start('90001');
+        });
+        (0, globals_1.test)('should return metadata', async () => {
+            (0, globals_1.expect)(metadata.errors).toBe(0);
+            (0, globals_1.expect)(metadata.properties).toBe(2);
+            (0, globals_1.expect)(metadata.requests).toBe(2);
         });
         (0, globals_1.test)('should convert existing field to captial-case', async () => {
             const result = await new Promise(resolve => {

package/dist/__tests__/setup.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.closeConnection = exports.openConnection = exports.setupProperties = void 0;
+exports.getFetchPropertiesForFix = exports.getFetchPropertiesForFilters = exports.closeConnection = exports.openConnection = exports.setupProperties = void 0;
 const mongodb_1 = require("mongodb");
 const mock_1 = require("./mock");
 async function setupProperties(base) {
@@ -10,6 +10,12 @@ async function setupProperties(base) {
     catch (e) {
         // ignore
     }
+    try {
+        await base.dropCollection('filters');
+    }
+    catch (e) {
+        // ignore
+    }
     try {
         await base.createCollection('properties');
         await base.collection('properties').insertMany(mock_1.MOCK);
@@ -33,3 +39,32 @@ const openConnection = async () => {
 exports.openConnection = openConnection;
 const closeConnection = () => connectedMongoClient?.close();
 exports.closeConnection = closeConnection;
+const getFetchPropertiesForFilters = (base) => async (zip) => {
+    return base
+        .collection('properties')
+        .find({ zipcode: zip }, {
+        projection: {
+            zipcode: 1,
+            propertyusegroup: 1,
+            propertyusestandardized: 1,
+            hvaccoolingdetail: 1,
+            hvacheatingdetail: 1,
+            utilitieswatersource: 1,
+            flooringmaterialprimary: 1,
+            _id: 0, // eslint-disable-line
+        },
+    })
+        .toArray();
+};
+exports.getFetchPropertiesForFilters = getFetchPropertiesForFilters;
+const getFetchPropertiesForFix = (base) => async (zip) => {
+    return base
+        .collection('properties')
+        .find({ zipcode: zip }, {
+        projection: {
+            propertyusegroup: 1,
+        },
+    })
+        .toArray();
+};
+exports.getFetchPropertiesForFix = getFetchPropertiesForFix;

package/dist/filter-fetcher/index.js

@@ -2,83 +2,56 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.getFilterFetcherProcessor = void 0;
 const calc_filter_count_1 = require("./calc-filter-count");
-function getFilterFetcherProcessor(db) {
-    const getPropertiesCollection = async () => {
-        return db.collection('properties');
-    };
+function getFilterFetcherProcessor({ mongoDb: db, fetchProperties }) {
     const getFiltersCollection = async () => {
         return db.collection('filters');
     };
-    const getProperties = async (zip) => {
-        const collection = await getPropertiesCollection();
-        return new Promise((resolve, reject) => {
-            collection
-                .find({ zipcode: zip }, {
-                projection: {
-                    zipcode: 1,
-                    propertyusegroup: 1,
-                    propertyusestandardized: 1,
-                    hvaccoolingdetail: 1,
-                    hvacheatingdetail: 1,
-                    utilitieswatersource: 1,
-                    flooringmaterialprimary: 1,
-                    _id: 0, // eslint-disable-line
-                },
-            })
-                .toArray((err, results) => {
-                if (err) {
-                    return reject(err);
-                }
-                resolve(results);
-            });
-        });
-    };
     const deleteZipFilter = async (zip) => {
         const collection = await getFiltersCollection();
-        return new Promise((resolve, reject) => {
-            collection.deleteOne({ zipcode: zip }, (err, results) => {
-                if (err) {
-                    return reject(err);
-                }
-                resolve(results);
-            });
-        });
+        return collection.deleteOne({ zipcode: zip });
     };
     const addZipFilters = async ({ zip, propertyusegroupCounts, propertyusestandardizedCounts, hvaccoolingdetailCounts, hvacheatingdetailCounts, utilitieswatersourceCounts, flooringmaterialprimaryCounts, }) => {
         const collection = await getFiltersCollection();
-        return new Promise((resolve, reject) => {
-            collection.insertOne({
-                zipcode: zip,
-                propertyusegroup: propertyusegroupCounts,
-                propertyusestandardized: propertyusestandardizedCounts,
-                hvaccoolingdetail: hvaccoolingdetailCounts,
-                hvacheatingdetail: hvacheatingdetailCounts,
-                utilitieswatersource: utilitieswatersourceCounts,
-                flooringmaterialprimary: flooringmaterialprimaryCounts,
-            }, (err, results) => {
-                if (err) {
-                    return reject(err);
-                }
-                resolve(results);
-            });
+        return collection.insertOne({
+            zipcode: zip,
+            propertyusegroup: propertyusegroupCounts,
+            propertyusestandardized: propertyusestandardizedCounts,
+            hvaccoolingdetail: hvaccoolingdetailCounts,
+            hvacheatingdetail: hvacheatingdetailCounts,
+            utilitieswatersource: utilitieswatersourceCounts,
+            flooringmaterialprimary: flooringmaterialprimaryCounts,
         });
     };
     return async function processZip(zip, errors) {
+        const meta = {
+            requests: 0,
+            errors: 0,
+            properties: 0,
+            filters: 0,
+        };
         if (!zip) {
-            return;
+            return meta;
         }
         try {
-            const properties = await getProperties(zip);
+            const properties = await fetchProperties(zip);
+            meta.requests += 1;
+            meta.properties += properties.length;
             const filters = (0, calc_filter_count_1.calculateFilterCount)(properties);
             await deleteZipFilter(zip);
+            meta.requests += 1;
             await addZipFilters({
                 zip,
                 ...filters,
             });
+            meta.filters += 1;
+            meta.requests += 1;
         }
         catch (e) {
+            console.error(e);
             errors.add(zip);
+            meta.errors += 1;
         }
+        return meta;
     };
 }
 exports.getFilterFetcherProcessor = getFilterFetcherProcessor;

package/dist/fix-property-use/index.js

@@ -8,24 +8,7 @@ const getPropertiesCollection = async (db) => {
 function fixPropertyName(propertyName) {
     return propertyName ? propertyName.toUpperCase() : '';
 }
-function getPropertyUseGroupProcessor(db) {
-    const getProperties = async (zip) => {
-        const collection = await getPropertiesCollection(db);
-        return new Promise((resolve, reject) => {
-            collection
-                .find({ zipcode: zip }, {
-                projection: {
-                    propertyusegroup: 1,
-                },
-            })
-                .toArray((err, results) => {
-                if (err) {
-                    return reject(err);
-                }
-                resolve(results);
-            });
-        });
-    };
+function getPropertyUseGroupProcessor({ mongoDb: db, fetchProperties }) {
     function fixPropertyNames(properties) {
         return properties
             .filter(p => p.propertyusegroup &&
@@ -50,23 +33,34 @@ function getPropertyUseGroupProcessor(db) {
         return collection.bulkWrite(propQuery, { ordered: true, w: 1 });
     };
     return async function processZip(zip, errors) {
+        const meta = {
+            requests: 0,
+            errors: 0,
+            properties: 0,
+        };
         if (!zip) {
-            return;
+            return meta;
         }
         try {
-            const properties = await getProperties(zip);
+            const properties = await fetchProperties(zip);
+            meta.requests += 1;
             if (!properties || !properties.length) {
-                return;
+                return meta;
             }
             const convertedProperties = fixPropertyNames(properties);
             if (!convertedProperties.length) {
-                return;
+                return meta;
             }
             await updateProperties(convertedProperties);
+            meta.properties += convertedProperties.length;
+            meta.requests += 1;
         }
         catch (e) {
+            console.error(e);
             errors.add(zip);
+            meta.errors += 1;
         }
+        return meta;
     };
 }
 exports.getPropertyUseGroupProcessor = getPropertyUseGroupProcessor;

package/dist/index.js

@@ -6,9 +6,9 @@ const fix_property_use_1 = require("./fix-property-use");
 const property_assessorlastsaledate_converter_1 = require("./property-assessorlastsaledate-converter");
 const zip_processor_1 = require("./zip-processor");
 function configureProcessor(config) {
-    const filterFetcher = (0, zip_processor_1.zipProcessor)(config, (0, filter_fetcher_1.getFilterFetcherProcessor)(config.mongoDb));
+    const filterFetcher = (0, zip_processor_1.zipProcessor)(config, (0, filter_fetcher_1.getFilterFetcherProcessor)(config));
     const assessorLastSaleDateProcessor = (0, zip_processor_1.zipProcessor)(config, (0, property_assessorlastsaledate_converter_1.getAssessorLastSaleDateProcessor)(config.mongoDb));
-    const propertyUseGroupProcessor = (0, zip_processor_1.zipProcessor)(config, (0, fix_property_use_1.getPropertyUseGroupProcessor)(config.mongoDb));
+    const propertyUseGroupProcessor = (0, zip_processor_1.zipProcessor)(config, (0, fix_property_use_1.getPropertyUseGroupProcessor)(config));
     return { filterFetcher, assessorLastSaleDateProcessor, propertyUseGroupProcessor };
 }
 exports.configureProcessor = configureProcessor;

package/dist/property-assessorlastsaledate-converter/index.js

@@ -8,26 +8,19 @@ const getPropertiesCollection = async (db) => {
 function getAssessorLastSaleDateProcessor(db) {
     const getPropertiesAssessorLastSaleDate = async (zip) => {
         const collection = await getPropertiesCollection(db);
-        return new Promise((resolve, reject) => {
-            collection
-                .find({
-                zipcode: zip,
-                assessorlastsaledate: {
-                    $exists: true,
-                    $type: 2,
-                },
-            }, {
-                projection: {
-                    assessorlastsaledate: 1,
-                },
-            })
-                .toArray((err, results) => {
-                if (err) {
-                    return reject(err);
-                }
-                resolve(results);
-            });
-        });
+        return collection
+            .find({
+            zipcode: zip,
+            assessorlastsaledate: {
+                $exists: true,
+                $type: 2,
+            },
+        }, {
+            projection: {
+                assessorlastsaledate: 1,
+            },
+        })
+            .toArray();
     };
     const updateProperties = async (properties) => {
         const collection = await getPropertiesCollection(db);
@@ -41,23 +34,34 @@ function getAssessorLastSaleDateProcessor(db) {
         return collection.bulkWrite(propQuery, { ordered: true, w: 1 });
     };
     return async function processZip(zip, errors) {
+        const meta = {
+            requests: 0,
+            errors: 0,
+            properties: 0,
+        };
         if (!zip) {
-            return;
+            return meta;
         }
         try {
             const properties = await getPropertiesAssessorLastSaleDate(zip);
+            meta.requests += 1;
             if (!properties || !properties.length) {
-                return;
+                return meta;
             }
             const convertedProperties = (0, utils_1.convertStringToDate)(properties, 'assessorlastsaledate');
             if (!convertedProperties.length) {
-                return;
+                return meta;
             }
             await updateProperties(convertedProperties);
+            meta.properties += convertedProperties.length;
+            meta.requests += 1;
         }
         catch (e) {
+            console.error(e);
+            meta.errors += 1;
             errors.add(zip);
         }
+        return meta;
     };
 }
 exports.getAssessorLastSaleDateProcessor = getAssessorLastSaleDateProcessor;

package/dist/zip-processor.js

@@ -10,19 +10,25 @@ function zipProcessor(config, processorFn) {
         if (!zips) {
             return;
         }
-        // eslint-disable-next-line prefer-const
-        let [startZipNum, endZipNum] = zips.split('-').map(Number);
-        if (!endZipNum) {
-            endZipNum = startZipNum;
-        }
-        for (let zipNum = startZipNum; zipNum <= endZipNum;) {
-            const processes = [];
-            const processZipEnd = zipNum + config.workers - 1;
-            for (; zipNum <= endZipNum && zipNum <= processZipEnd; zipNum++) {
-                const zip = (0, utils_1.convertToZipString)(zipNum);
-                processes.push(processorFn(zip, erroredZips));
+        if (Array.isArray(zips)) {
+            // eslint-disable-next-line prefer-const
+            let [startZipNum, endZipNum] = zips.map(Number);
+            if (!endZipNum) {
+                endZipNum = startZipNum;
             }
-            yield await Promise.all(processes);
+            for (let zipNum = startZipNum; zipNum <= endZipNum;) {
+                const processes = [];
+                const processZipEnd = zipNum + config.workers - 1;
+                for (; zipNum <= endZipNum && zipNum <= processZipEnd; zipNum++) {
+                    const zip = (0, utils_1.convertToZipString)(zipNum);
+                    processes.push(processorFn(zip, erroredZips));
+                }
+                yield await Promise.all(processes);
+            }
+        }
+        else {
+            const zip = (0, utils_1.convertToZipString)(Number(zips));
+            yield await processorFn(zip, erroredZips);
         }
         while (erroredZips.size) {
             const connectionErrorZipsInner = [...erroredZips];
@@ -39,17 +45,39 @@ function zipProcessor(config, processorFn) {
         }
         processor = null;
     }
+    const meta = {
+        requests: 0,
+        properties: 0,
+        errors: 0,
+        filters: 0,
+    };
+    function sumMetas(metas) {
+        if (!metas) {
+            return;
+        }
+        return (Array.isArray(metas) ? metas : [metas]).forEach(m => {
+            meta.requests += m.requests;
+            meta.properties += m.properties;
+            meta.errors += m.errors;
+            if (m.filters) {
+                meta.filters += m.filters;
+            }
+        });
+    }
     async function start(zips) {
         processor = processZips(zips);
         try {
-            let step = await processor.next();
-            while (processor && !step.done) {
-                step = await processor.next();
+            let isDone = false;
+            while (processor && !isDone) {
+                const step = await processor.next();
+                sumMetas(step.value);
+                isDone = Boolean(step.done);
             }
         }
         catch (e) {
             await clear();
         }
+        return meta;
     }
     return {
         getProcessor: () => processor,

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@servicetitan/acquisition-functions",
-    "version": "0.1.0",
+    "version": "0.3.0",
     "description": "",
     "main": "./dist/index.js",
     "typings": "./dist/index.d.ts",
@@ -28,5 +28,5 @@
     "cli": {
         "webpack": false
     },
-    "gitHead": "30de32ace907277b336c7bc835e659b2a3ffaac9"
+    "gitHead": "c4cdbc238e286e33a9eea2b8945e2820b137b916"
 }