@servicetitan/acquisition-functions 0.1.0 → 0.2.0

package/README.md

@@ -33,9 +33,5 @@ npm run test
 In Docker:
 
 ```sh
-# In root FE directory
-lerna run build --scope=@servicetitan/acquisition-functions
-
-# In package directory
-docker-compose up
+docker-compose up --from-exit-code=tests
 ```

package/dist/__tests__/assessorlastsaledate-converter.test.js

@@ -6,6 +6,7 @@ const setup_1 = require("./setup");
 describe('property-assessorlastsaledate-converter', () => {
     let processor;
     let base;
+    let metadata;
     (0, globals_1.beforeAll)(async () => {
         const client = await (0, setup_1.openConnection)();
         base = client.db('test');
@@ -18,7 +19,7 @@ describe('property-assessorlastsaledate-converter', () => {
                 mongoDb: base,
             });
             const { assessorLastSaleDateProcessor } = processor;
-            await assessorLastSaleDateProcessor.start('90001');
+            metadata = await assessorLastSaleDateProcessor.start('90001');
         });
         (0, globals_1.test)('should convert existing field to date', async () => {
             const result = await new Promise(resolve => {
@@ -46,6 +47,11 @@ describe('property-assessorlastsaledate-converter', () => {
             });
             (0, globals_1.expect)(result[0].assessorlastsaledate).toBe(null);
         });
+        (0, globals_1.test)('should return metadata', async () => {
+            (0, globals_1.expect)(metadata.errors).toBe(0);
+            (0, globals_1.expect)(metadata.properties).toBe(1);
+            (0, globals_1.expect)(metadata.requests).toBe(2);
+        });
     });
     describe('aggregate', () => {
         (0, globals_1.beforeEach)(async () => {

package/dist/__tests__/filter-fetcher.test.js

@@ -6,6 +6,7 @@ const setup_1 = require("./setup");
 describe('property-assessorlastsaledate-converter', () => {
     let processor;
     let base;
+    let metadata;
     (0, globals_1.beforeAll)(async () => {
         const client = await (0, setup_1.openConnection)();
         base = client.db('test');
@@ -16,7 +17,7 @@ describe('property-assessorlastsaledate-converter', () => {
         });
         const { filterFetcher, propertyUseGroupProcessor } = processor;
         await propertyUseGroupProcessor.start('90001');
-        await filterFetcher.start('90001-90002');
+        metadata = await filterFetcher.start(['90001', '90002']);
     });
     (0, globals_1.test)('should have empty filter for missing zip', async () => {
         const result = await base.collection('filters').findOne({
@@ -30,6 +31,12 @@ describe('property-assessorlastsaledate-converter', () => {
         (0, globals_1.expect)(result?.propertyusestandardized.length).toBe(0);
         (0, globals_1.expect)(result?.utilitieswatersource.length).toBe(0);
     });
+    (0, globals_1.test)('should return metadata', async () => {
+        (0, globals_1.expect)(metadata.errors).toBe(0);
+        (0, globals_1.expect)(metadata.properties).toBe(2);
+        (0, globals_1.expect)(metadata.requests).toBe(6);
+        (0, globals_1.expect)(metadata.filters).toBe(2);
+    });
     (0, globals_1.test)('should form correct filter', async () => {
         const result = await base.collection('filters').findOne({
             zipcode: '90001',

package/dist/__tests__/property-fix-use-group.test.js

@@ -11,6 +11,7 @@ describe('property-use-group-converter', () => {
     });
     describe('processor', () => {
         let processor;
+        let metadata;
         (0, globals_1.beforeAll)(async () => {
             await (0, setup_1.setupProperties)(base);
             processor = (0, index_1.configureProcessor)({
@@ -18,7 +19,12 @@ describe('property-use-group-converter', () => {
                 mongoDb: base,
             });
             const { propertyUseGroupProcessor } = processor;
-            await propertyUseGroupProcessor.start('90001-90002');
+            metadata = await propertyUseGroupProcessor.start('90001');
+        });
+        (0, globals_1.test)('should return metadata', async () => {
+            (0, globals_1.expect)(metadata.errors).toBe(0);
+            (0, globals_1.expect)(metadata.properties).toBe(2);
+            (0, globals_1.expect)(metadata.requests).toBe(2);
         });
         (0, globals_1.test)('should convert existing field to captial-case', async () => {
             const result = await new Promise(resolve => {

package/dist/__tests__/setup.js

@@ -10,6 +10,12 @@ async function setupProperties(base) {
     catch (e) {
         // ignore
     }
+    try {
+        await base.dropCollection('filters');
+    }
+    catch (e) {
+        // ignore
+    }
     try {
         await base.createCollection('properties');
         await base.collection('properties').insertMany(mock_1.MOCK);

package/dist/filter-fetcher/index.js

@@ -11,74 +11,67 @@ function getFilterFetcherProcessor(db) {
     };
     const getProperties = async (zip) => {
         const collection = await getPropertiesCollection();
-        return new Promise((resolve, reject) => {
-            collection
-                .find({ zipcode: zip }, {
-                projection: {
-                    zipcode: 1,
-                    propertyusegroup: 1,
-                    propertyusestandardized: 1,
-                    hvaccoolingdetail: 1,
-                    hvacheatingdetail: 1,
-                    utilitieswatersource: 1,
-                    flooringmaterialprimary: 1,
-                    _id: 0, // eslint-disable-line
-                },
-            })
-                .toArray((err, results) => {
-                if (err) {
-                    return reject(err);
-                }
-                resolve(results);
-            });
-        });
+        return collection
+            .find({ zipcode: zip }, {
+            projection: {
+                zipcode: 1,
+                propertyusegroup: 1,
+                propertyusestandardized: 1,
+                hvaccoolingdetail: 1,
+                hvacheatingdetail: 1,
+                utilitieswatersource: 1,
+                flooringmaterialprimary: 1,
+                _id: 0, // eslint-disable-line
+            },
+        })
+            .toArray();
     };
     const deleteZipFilter = async (zip) => {
         const collection = await getFiltersCollection();
-        return new Promise((resolve, reject) => {
-            collection.deleteOne({ zipcode: zip }, (err, results) => {
-                if (err) {
-                    return reject(err);
-                }
-                resolve(results);
-            });
-        });
+        return collection.deleteOne({ zipcode: zip });
     };
     const addZipFilters = async ({ zip, propertyusegroupCounts, propertyusestandardizedCounts, hvaccoolingdetailCounts, hvacheatingdetailCounts, utilitieswatersourceCounts, flooringmaterialprimaryCounts, }) => {
         const collection = await getFiltersCollection();
-        return new Promise((resolve, reject) => {
-            collection.insertOne({
-                zipcode: zip,
-                propertyusegroup: propertyusegroupCounts,
-                propertyusestandardized: propertyusestandardizedCounts,
-                hvaccoolingdetail: hvaccoolingdetailCounts,
-                hvacheatingdetail: hvacheatingdetailCounts,
-                utilitieswatersource: utilitieswatersourceCounts,
-                flooringmaterialprimary: flooringmaterialprimaryCounts,
-            }, (err, results) => {
-                if (err) {
-                    return reject(err);
-                }
-                resolve(results);
-            });
+        return collection.insertOne({
+            zipcode: zip,
+            propertyusegroup: propertyusegroupCounts,
+            propertyusestandardized: propertyusestandardizedCounts,
+            hvaccoolingdetail: hvaccoolingdetailCounts,
+            hvacheatingdetail: hvacheatingdetailCounts,
+            utilitieswatersource: utilitieswatersourceCounts,
+            flooringmaterialprimary: flooringmaterialprimaryCounts,
         });
     };
     return async function processZip(zip, errors) {
+        const meta = {
+            requests: 0,
+            errors: 0,
+            properties: 0,
+            filters: 0,
+        };
         if (!zip) {
-            return;
+            return meta;
         }
         try {
             const properties = await getProperties(zip);
+            meta.requests += 1;
+            meta.properties += properties.length;
             const filters = (0, calc_filter_count_1.calculateFilterCount)(properties);
             await deleteZipFilter(zip);
+            meta.requests += 1;
             await addZipFilters({
                 zip,
                 ...filters,
             });
+            meta.filters += 1;
+            meta.requests += 1;
         }
         catch (e) {
+            console.error(e);
             errors.add(zip);
+            meta.errors += 1;
         }
+        return meta;
     };
 }
 exports.getFilterFetcherProcessor = getFilterFetcherProcessor;

package/dist/fix-property-use/index.js

@@ -11,20 +11,13 @@ function fixPropertyName(propertyName) {
 function getPropertyUseGroupProcessor(db) {
     const getProperties = async (zip) => {
         const collection = await getPropertiesCollection(db);
-        return new Promise((resolve, reject) => {
-            collection
-                .find({ zipcode: zip }, {
-                projection: {
-                    propertyusegroup: 1,
-                },
-            })
-                .toArray((err, results) => {
-                if (err) {
-                    return reject(err);
-                }
-                resolve(results);
-            });
-        });
+        return collection
+            .find({ zipcode: zip }, {
+            projection: {
+                propertyusegroup: 1,
+            },
+        })
+            .toArray();
     };
     function fixPropertyNames(properties) {
         return properties
@@ -50,23 +43,34 @@ function getPropertyUseGroupProcessor(db) {
         return collection.bulkWrite(propQuery, { ordered: true, w: 1 });
     };
     return async function processZip(zip, errors) {
+        const meta = {
+            requests: 0,
+            errors: 0,
+            properties: 0,
+        };
         if (!zip) {
-            return;
+            return meta;
         }
         try {
             const properties = await getProperties(zip);
+            meta.requests += 1;
             if (!properties || !properties.length) {
-                return;
+                return meta;
             }
             const convertedProperties = fixPropertyNames(properties);
             if (!convertedProperties.length) {
-                return;
+                return meta;
             }
             await updateProperties(convertedProperties);
+            meta.properties += convertedProperties.length;
+            meta.requests += 1;
         }
         catch (e) {
+            console.error(e);
             errors.add(zip);
+            meta.errors += 1;
         }
+        return meta;
     };
 }
 exports.getPropertyUseGroupProcessor = getPropertyUseGroupProcessor;

package/dist/property-assessorlastsaledate-converter/index.js

@@ -8,26 +8,19 @@ const getPropertiesCollection = async (db) => {
 function getAssessorLastSaleDateProcessor(db) {
     const getPropertiesAssessorLastSaleDate = async (zip) => {
         const collection = await getPropertiesCollection(db);
-        return new Promise((resolve, reject) => {
-            collection
-                .find({
-                zipcode: zip,
-                assessorlastsaledate: {
-                    $exists: true,
-                    $type: 2,
-                },
-            }, {
-                projection: {
-                    assessorlastsaledate: 1,
-                },
-            })
-                .toArray((err, results) => {
-                if (err) {
-                    return reject(err);
-                }
-                resolve(results);
-            });
-        });
+        return collection
+            .find({
+            zipcode: zip,
+            assessorlastsaledate: {
+                $exists: true,
+                $type: 2,
+            },
+        }, {
+            projection: {
+                assessorlastsaledate: 1,
+            },
+        })
+            .toArray();
     };
     const updateProperties = async (properties) => {
         const collection = await getPropertiesCollection(db);
@@ -41,23 +34,34 @@ function getAssessorLastSaleDateProcessor(db) {
         return collection.bulkWrite(propQuery, { ordered: true, w: 1 });
     };
     return async function processZip(zip, errors) {
+        const meta = {
+            requests: 0,
+            errors: 0,
+            properties: 0,
+        };
         if (!zip) {
-            return;
+            return meta;
         }
         try {
             const properties = await getPropertiesAssessorLastSaleDate(zip);
+            meta.requests += 1;
             if (!properties || !properties.length) {
-                return;
+                return meta;
             }
             const convertedProperties = (0, utils_1.convertStringToDate)(properties, 'assessorlastsaledate');
             if (!convertedProperties.length) {
-                return;
+                return meta;
             }
             await updateProperties(convertedProperties);
+            meta.properties += convertedProperties.length;
+            meta.requests += 1;
         }
         catch (e) {
+            console.error(e);
+            meta.errors += 1;
             errors.add(zip);
         }
+        return meta;
     };
 }
 exports.getAssessorLastSaleDateProcessor = getAssessorLastSaleDateProcessor;

package/dist/zip-processor.js

@@ -10,19 +10,25 @@ function zipProcessor(config, processorFn) {
         if (!zips) {
             return;
         }
-        // eslint-disable-next-line prefer-const
-        let [startZipNum, endZipNum] = zips.split('-').map(Number);
-        if (!endZipNum) {
-            endZipNum = startZipNum;
-        }
-        for (let zipNum = startZipNum; zipNum <= endZipNum;) {
-            const processes = [];
-            const processZipEnd = zipNum + config.workers - 1;
-            for (; zipNum <= endZipNum && zipNum <= processZipEnd; zipNum++) {
-                const zip = (0, utils_1.convertToZipString)(zipNum);
-                processes.push(processorFn(zip, erroredZips));
+        if (Array.isArray(zips)) {
+            // eslint-disable-next-line prefer-const
+            let [startZipNum, endZipNum] = zips.map(Number);
+            if (!endZipNum) {
+                endZipNum = startZipNum;
             }
-            yield await Promise.all(processes);
+            for (let zipNum = startZipNum; zipNum <= endZipNum;) {
+                const processes = [];
+                const processZipEnd = zipNum + config.workers - 1;
+                for (; zipNum <= endZipNum && zipNum <= processZipEnd; zipNum++) {
+                    const zip = (0, utils_1.convertToZipString)(zipNum);
+                    processes.push(processorFn(zip, erroredZips));
+                }
+                yield await Promise.all(processes);
+            }
+        }
+        else {
+            const zip = (0, utils_1.convertToZipString)(Number(zips));
+            yield await processorFn(zip, erroredZips);
         }
         while (erroredZips.size) {
             const connectionErrorZipsInner = [...erroredZips];
@@ -39,17 +45,39 @@ function zipProcessor(config, processorFn) {
         }
         processor = null;
     }
+    const meta = {
+        requests: 0,
+        properties: 0,
+        errors: 0,
+        filters: 0,
+    };
+    function sumMetas(metas) {
+        if (!metas) {
+            return;
+        }
+        return (Array.isArray(metas) ? metas : [metas]).forEach(m => {
+            meta.requests += m.requests;
+            meta.properties += m.properties;
+            meta.errors += m.errors;
+            if (m.filters) {
+                meta.filters += m.filters;
+            }
+        });
+    }
     async function start(zips) {
         processor = processZips(zips);
         try {
-            let step = await processor.next();
-            while (processor && !step.done) {
-                step = await processor.next();
+            let isDone = false;
+            while (processor && !isDone) {
+                const step = await processor.next();
+                sumMetas(step.value);
+                isDone = Boolean(step.done);
             }
         }
         catch (e) {
             await clear();
         }
+        return meta;
     }
     return {
         getProcessor: () => processor,

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@servicetitan/acquisition-functions",
-    "version": "0.1.0",
+    "version": "0.2.0",
     "description": "",
     "main": "./dist/index.js",
     "typings": "./dist/index.d.ts",
@@ -28,5 +28,5 @@
     "cli": {
         "webpack": false
     },
-    "gitHead": "30de32ace907277b336c7bc835e659b2a3ffaac9"
+    "gitHead": "7c93d00316dcd4bd83e1a42720579ed22c6ce208"
 }