incyclist-services 1.7.66 → 1.7.67

package/lib/cjs/activities/base/utils/activity.js

@@ -181,7 +181,7 @@ let Activity = (() => {
             try {
                 const converter = this.getActivityConverter();
                 const data = await converter.convert(this.details, format);
-                const fileName = this.details.fileName.replace('.json', `.${format}`);
+                const fileName = await this.getExportFileName(format);
                 await fs.writeFile(fileName, Buffer.from(data));
                 this.details[`${format}FileName`] = fileName;
                 await this.save(true);
@@ -197,6 +197,18 @@ let Activity = (() => {
             }
             return success;
         }
+        async getExportFileName(format) {
+            let fileName = this.details.fileName.replace('json', format);
+            if (fileName.startsWith('mmkv:/')) {
+                const fs = this.getBindings().fs;
+                const appDir = this.getBindings().appInfo.getAppDir();
+                const activitiesDir = this.getBindings().path.join(appDir, 'activities');
+                await fs.ensureDir(activitiesDir);
+                const baseName = fileName.split('/').pop();
+                fileName = this.getBindings().path.join(activitiesDir, baseName);
+            }
+            return fileName;
+        }
         async upload(connectedApp, observer) {
             let success = false;
             let error;

package/lib/cjs/services/overpass/overpass.js

@@ -1,4 +1,11 @@
 "use strict";
+var __runInitializers = (this && this.__runInitializers) || function (thisArg, initializers, value) {
+    var useValue = arguments.length > 2;
+    for (var i = 0; i < initializers.length; i++) {
+        value = useValue ? initializers[i].call(thisArg, value) : initializers[i].call(thisArg);
+    }
+    return useValue ? value : void 0;
+};
 var __esDecorate = (this && this.__esDecorate) || function (ctor, descriptorIn, decorators, contextIn, initializers, extraInitializers) {
     function accept(f) { if (f !== void 0 && typeof f !== "function") throw new TypeError("Function expected"); return f; }
     var kind = contextIn.kind, key = kind === "getter" ? "get" : kind === "setter" ? "set" : "value";
@@ -26,13 +33,6 @@ var __esDecorate = (this && this.__esDecorate) || function (ctor, descriptorIn,
     if (target) Object.defineProperty(target, contextIn.name, descriptor);
     done = true;
 };
-var __runInitializers = (this && this.__runInitializers) || function (thisArg, initializers, value) {
-    var useValue = arguments.length > 2;
-    for (var i = 0; i < initializers.length; i++) {
-        value = useValue ? initializers[i].call(thisArg, value) : initializers[i].call(thisArg);
-    }
-    return useValue ? value : void 0;
-};
 var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
@@ -43,6 +43,9 @@ const types_1 = require("../../base/types");
 const sleep_1 = require("../../utils/sleep");
 const base_1 = require("../../apps/base/api/base");
 const gd_eventlog_1 = require("gd-eventlog");
+const decorators_1 = require("../../base/decorators");
+const api_1 = require("../../api");
+const settings_1 = require("../../settings");
 const OVERPASS_URL_ALT1 = 'https://overpass.kumi.systems/api/interpreter';
 const OVERPASS_URL_ALT2 = 'https://lz4.overpass-api.de/api/interpreter';
 const OVERPASS_URL_ALT3 = 'https://z.overpass-api.de/api/interpreter';
@@ -53,16 +56,23 @@ let OverpassApi = (() => {
     let _classExtraInitializers = [];
     let _classThis;
     let _classSuper = base_1.AppApiBase;
+    let _instanceExtraInitializers = [];
+    let _getUserSettings_decorators;
+    let _getBindings_decorators;
     var OverpassApi = class extends _classSuper {
         static { _classThis = this; }
         static {
             const _metadata = typeof Symbol === "function" && Symbol.metadata ? Object.create(_classSuper[Symbol.metadata] ?? null) : void 0;
+            _getUserSettings_decorators = [decorators_1.Injectable];
+            _getBindings_decorators = [decorators_1.Injectable];
+            __esDecorate(this, null, _getUserSettings_decorators, { kind: "method", name: "getUserSettings", static: false, private: false, access: { has: obj => "getUserSettings" in obj, get: obj => obj.getUserSettings }, metadata: _metadata }, null, _instanceExtraInitializers);
+            __esDecorate(this, null, _getBindings_decorators, { kind: "method", name: "getBindings", static: false, private: false, access: { has: obj => "getBindings" in obj, get: obj => obj.getBindings }, metadata: _metadata }, null, _instanceExtraInitializers);
             __esDecorate(null, _classDescriptor = { value: _classThis }, _classDecorators, { kind: "class", name: _classThis.name, metadata: _metadata }, null, _classExtraInitializers);
             OverpassApi = _classThis = _classDescriptor.value;
             if (_metadata) Object.defineProperty(_classThis, Symbol.metadata, { enumerable: true, configurable: true, writable: true, value: _metadata });
             __runInitializers(_classThis, _classExtraInitializers);
         }
-        url;
+        url = __runInitializers(this, _instanceExtraInitializers);
         mirrors = [];
         logger;
         constructor(props) {
@@ -76,6 +86,70 @@ let OverpassApi = (() => {
                 }
             }
         }
+        getOverpassHeaders() {
+            const version = this.getAppVersion();
+            const userId = this.getUserId();
+            let userAgent = `Incyclist/${version}`;
+            if (userId) {
+                try {
+                    const crypto = this.getBindings()?.crypto;
+                    const hashedId = crypto ? crypto.createHash('sha256').update('OAPI:' + userId).digest('hex').substring(0, 8) : undefined;
+                    if (hashedId) {
+                        userAgent += ` (${hashedId})`;
+                    }
+                }
+                catch (err) {
+                    this.logger.logEvent({ message: 'failed to hash userId', error: err.message });
+                }
+            }
+            return {
+                'User-Agent': userAgent,
+                'Referer': 'https://incyclist.com'
+            };
+        }
+        async post(url, data, conf) {
+            try {
+                const https = require('https');
+                return await this.postViaNodeHttps(https, url, data);
+            }
+            catch {
+                const cfg = conf ?? {};
+                const config = {
+                    ...cfg,
+                    headers: this.getOverpassHeaders()
+                };
+                return await super.post(url, data, config);
+            }
+        }
+        async postViaNodeHttps(https, url, data) {
+            return new Promise((resolve, reject) => {
+                const reqUrl = new URL(url);
+                const headers = this.getOverpassHeaders();
+                const queryData = typeof data === 'string' ? data : JSON.stringify(data);
+                const options = {
+                    method: 'POST',
+                    headers: {
+                        ...headers,
+                        'Content-Length': Buffer.byteLength(queryData)
+                    }
+                };
+                const req = https.request(reqUrl, options, (res) => {
+                    let body = '';
+                    res.on('data', (chunk) => body += chunk);
+                    res.on('end', () => {
+                        resolve({
+                            data: body,
+                            status: res.statusCode,
+                            statusText: res.statusMessage,
+                            headers: res.headers
+                        });
+                    });
+                });
+                req.on('error', reject);
+                req.write(queryData);
+                req.end();
+            });
+        }
         async query(queryOL, timeout) {
             const ts = Date.now();
             this.logger.logEvent({ message: 'query', queryOL, queryId: ts, timeout });
@@ -96,24 +170,102 @@ let OverpassApi = (() => {
         getBaseUrl() {
             return '';
         }
+        extractErrorInfo(err, mirror) {
+            return {
+                mirror,
+                status: err?.response?.status,
+                statusText: err?.response?.statusText,
+                error: err?.message ?? 'overpass query failed'
+            };
+        }
+        logUnexpectedErrors(fn, errors) {
+            const unexpectedErrors = errors.filter(e => {
+                const status = e.status;
+                if (!status || status === 408 || status === 504)
+                    return false;
+                return status >= 406 && status < 500;
+            });
+            if (unexpectedErrors.length > 0 && unexpectedErrors.length === errors.length) {
+                unexpectedErrors.forEach(err => {
+                    this.logger.logEvent({
+                        message: 'error',
+                        fn,
+                        mirror: err.mirror,
+                        status: err.status,
+                        statusText: err.statusText,
+                        error: err.error ?? 'all mirrors failed with 4xx error'
+                    });
+                });
+            }
+        }
         async bulkQuery(query, timeout) {
+            const errors = [];
             const promises = [];
             this.mirrors.forEach(mirror => {
-                promises.push(this.post(mirror, query).then((res) => res?.data));
+                promises.push(this.post(mirror, query)
+                    .then((res) => res?.data)
+                    .catch(err => {
+                    errors.push(this.extractErrorInfo(err, mirror));
+                    throw err;
+                }));
             });
             if (timeout !== undefined && timeout !== null) {
                 promises.push((0, sleep_1.sleep)(timeout).then(() => 'timeout'));
             }
-            const res = await (0, promise_any_1.default)(promises);
-            if (res === 'timeout') {
-                return undefined;
+            try {
+                const res = await (0, promise_any_1.default)(promises);
+                if (res === 'timeout') {
+                    return undefined;
+                }
+                return res;
+            }
+            catch (err) {
+                this.logUnexpectedErrors('bulkQuery', errors);
+                throw err;
             }
-            return res;
+        }
+        async sequentialQuery(query, timeout) {
+            const startTime = Date.now();
+            const errors = [];
+            let lastError;
+            for (const mirror of this.mirrors) {
+                if (timeout !== undefined && timeout !== null) {
+                    const elapsed = Date.now() - startTime;
+                    if (elapsed >= timeout) {
+                        return undefined;
+                    }
+                }
+                try {
+                    const res = await this.post(mirror, query);
+                    return res?.data;
+                }
+                catch (err) {
+                    lastError = err;
+                    errors.push(this.extractErrorInfo(err, mirror));
+                    if (mirror !== this.mirrors[this.mirrors.length - 1]) {
+                        await (0, sleep_1.sleep)(100);
+                    }
+                }
+            }
+            this.logUnexpectedErrors('sequentialQuery', errors);
+            throw lastError;
         }
         reset() {
             this.mirrors = [OVERPASS_URL_ALT1, OVERPASS_URL_ALT2, OVERPASS_URL_ALT3];
             this.url = this.mirrors[0];
         }
+        getAppVersion() {
+            return this.getBindings()?.appInfo?.getAppVersion() ?? '';
+        }
+        getUserId() {
+            return this.getUserSettings().getValue('uuid', '');
+        }
+        getUserSettings() {
+            return (0, settings_1.useUserSettings)();
+        }
+        getBindings() {
+            return (0, api_1.getBindings)();
+        }
     };
     return OverpassApi = _classThis;
 })();

package/lib/esm/activities/base/utils/activity.js

@@ -178,7 +178,7 @@ let Activity = (() => {
             try {
                 const converter = this.getActivityConverter();
                 const data = await converter.convert(this.details, format);
-                const fileName = this.details.fileName.replace('.json', `.${format}`);
+                const fileName = await this.getExportFileName(format);
                 await fs.writeFile(fileName, Buffer.from(data));
                 this.details[`${format}FileName`] = fileName;
                 await this.save(true);
@@ -194,6 +194,18 @@ let Activity = (() => {
             }
             return success;
         }
+        async getExportFileName(format) {
+            let fileName = this.details.fileName.replace('json', format);
+            if (fileName.startsWith('mmkv:/')) {
+                const fs = this.getBindings().fs;
+                const appDir = this.getBindings().appInfo.getAppDir();
+                const activitiesDir = this.getBindings().path.join(appDir, 'activities');
+                await fs.ensureDir(activitiesDir);
+                const baseName = fileName.split('/').pop();
+                fileName = this.getBindings().path.join(activitiesDir, baseName);
+            }
+            return fileName;
+        }
         async upload(connectedApp, observer) {
             let success = false;
             let error;

package/lib/esm/package.json

@@ -1,6 +1,6 @@
 {
     "name": "incyclist-services",
-    "version": "1.7.66",
+    "version": "1.7.65",
     "peerDependencies": {
         "gd-eventlog": "^0.1.27"
     },

package/lib/esm/services/overpass/overpass.js

@@ -1,3 +1,10 @@
+var __runInitializers = (this && this.__runInitializers) || function (thisArg, initializers, value) {
+    var useValue = arguments.length > 2;
+    for (var i = 0; i < initializers.length; i++) {
+        value = useValue ? initializers[i].call(thisArg, value) : initializers[i].call(thisArg);
+    }
+    return useValue ? value : void 0;
+};
 var __esDecorate = (this && this.__esDecorate) || function (ctor, descriptorIn, decorators, contextIn, initializers, extraInitializers) {
     function accept(f) { if (f !== void 0 && typeof f !== "function") throw new TypeError("Function expected"); return f; }
     var kind = contextIn.kind, key = kind === "getter" ? "get" : kind === "setter" ? "set" : "value";
@@ -25,18 +32,14 @@ var __esDecorate = (this && this.__esDecorate) || function (ctor, descriptorIn,
     if (target) Object.defineProperty(target, contextIn.name, descriptor);
     done = true;
 };
-var __runInitializers = (this && this.__runInitializers) || function (thisArg, initializers, value) {
-    var useValue = arguments.length > 2;
-    for (var i = 0; i < initializers.length; i++) {
-        value = useValue ? initializers[i].call(thisArg, value) : initializers[i].call(thisArg);
-    }
-    return useValue ? value : void 0;
-};
 import any from 'promise.any';
 import { Singleton } from '../../base/types';
 import { sleep } from '../../utils/sleep';
 import { AppApiBase } from '../../apps/base/api/base';
 import { EventLogger } from 'gd-eventlog';
+import { Injectable } from '../../base/decorators';
+import { getBindings } from '../../api';
+import { useUserSettings } from '../../settings';
 const OVERPASS_URL_ALT1 = 'https://overpass.kumi.systems/api/interpreter';
 const OVERPASS_URL_ALT2 = 'https://lz4.overpass-api.de/api/interpreter';
 const OVERPASS_URL_ALT3 = 'https://z.overpass-api.de/api/interpreter';
@@ -47,16 +50,23 @@ let OverpassApi = (() => {
     let _classExtraInitializers = [];
     let _classThis;
     let _classSuper = AppApiBase;
+    let _instanceExtraInitializers = [];
+    let _getUserSettings_decorators;
+    let _getBindings_decorators;
     var OverpassApi = class extends _classSuper {
         static { _classThis = this; }
         static {
             const _metadata = typeof Symbol === "function" && Symbol.metadata ? Object.create(_classSuper[Symbol.metadata] ?? null) : void 0;
+            _getUserSettings_decorators = [Injectable];
+            _getBindings_decorators = [Injectable];
+            __esDecorate(this, null, _getUserSettings_decorators, { kind: "method", name: "getUserSettings", static: false, private: false, access: { has: obj => "getUserSettings" in obj, get: obj => obj.getUserSettings }, metadata: _metadata }, null, _instanceExtraInitializers);
+            __esDecorate(this, null, _getBindings_decorators, { kind: "method", name: "getBindings", static: false, private: false, access: { has: obj => "getBindings" in obj, get: obj => obj.getBindings }, metadata: _metadata }, null, _instanceExtraInitializers);
             __esDecorate(null, _classDescriptor = { value: _classThis }, _classDecorators, { kind: "class", name: _classThis.name, metadata: _metadata }, null, _classExtraInitializers);
             OverpassApi = _classThis = _classDescriptor.value;
             if (_metadata) Object.defineProperty(_classThis, Symbol.metadata, { enumerable: true, configurable: true, writable: true, value: _metadata });
             __runInitializers(_classThis, _classExtraInitializers);
         }
-        url;
+        url = __runInitializers(this, _instanceExtraInitializers);
         mirrors = [];
         logger;
         constructor(props) {
@@ -70,6 +80,70 @@ let OverpassApi = (() => {
                 }
             }
         }
+        getOverpassHeaders() {
+            const version = this.getAppVersion();
+            const userId = this.getUserId();
+            let userAgent = `Incyclist/${version}`;
+            if (userId) {
+                try {
+                    const crypto = this.getBindings()?.crypto;
+                    const hashedId = crypto ? crypto.createHash('sha256').update('OAPI:' + userId).digest('hex').substring(0, 8) : undefined;
+                    if (hashedId) {
+                        userAgent += ` (${hashedId})`;
+                    }
+                }
+                catch (err) {
+                    this.logger.logEvent({ message: 'failed to hash userId', error: err.message });
+                }
+            }
+            return {
+                'User-Agent': userAgent,
+                'Referer': 'https://incyclist.com'
+            };
+        }
+        async post(url, data, conf) {
+            try {
+                const https = require('https');
+                return await this.postViaNodeHttps(https, url, data);
+            }
+            catch {
+                const cfg = conf ?? {};
+                const config = {
+                    ...cfg,
+                    headers: this.getOverpassHeaders()
+                };
+                return await super.post(url, data, config);
+            }
+        }
+        async postViaNodeHttps(https, url, data) {
+            return new Promise((resolve, reject) => {
+                const reqUrl = new URL(url);
+                const headers = this.getOverpassHeaders();
+                const queryData = typeof data === 'string' ? data : JSON.stringify(data);
+                const options = {
+                    method: 'POST',
+                    headers: {
+                        ...headers,
+                        'Content-Length': Buffer.byteLength(queryData)
+                    }
+                };
+                const req = https.request(reqUrl, options, (res) => {
+                    let body = '';
+                    res.on('data', (chunk) => body += chunk);
+                    res.on('end', () => {
+                        resolve({
+                            data: body,
+                            status: res.statusCode,
+                            statusText: res.statusMessage,
+                            headers: res.headers
+                        });
+                    });
+                });
+                req.on('error', reject);
+                req.write(queryData);
+                req.end();
+            });
+        }
         async query(queryOL, timeout) {
             const ts = Date.now();
             this.logger.logEvent({ message: 'query', queryOL, queryId: ts, timeout });
@@ -90,24 +164,102 @@ let OverpassApi = (() => {
         getBaseUrl() {
             return '';
         }
+        extractErrorInfo(err, mirror) {
+            return {
+                mirror,
+                status: err?.response?.status,
+                statusText: err?.response?.statusText,
+                error: err?.message ?? 'overpass query failed'
+            };
+        }
+        logUnexpectedErrors(fn, errors) {
+            const unexpectedErrors = errors.filter(e => {
+                const status = e.status;
+                if (!status || status === 408 || status === 504)
+                    return false;
+                return status >= 406 && status < 500;
+            });
+            if (unexpectedErrors.length > 0 && unexpectedErrors.length === errors.length) {
+                unexpectedErrors.forEach(err => {
+                    this.logger.logEvent({
+                        message: 'error',
+                        fn,
+                        mirror: err.mirror,
+                        status: err.status,
+                        statusText: err.statusText,
+                        error: err.error ?? 'all mirrors failed with 4xx error'
+                    });
+                });
+            }
+        }
         async bulkQuery(query, timeout) {
+            const errors = [];
             const promises = [];
             this.mirrors.forEach(mirror => {
-                promises.push(this.post(mirror, query).then((res) => res?.data));
+                promises.push(this.post(mirror, query)
+                    .then((res) => res?.data)
+                    .catch(err => {
+                    errors.push(this.extractErrorInfo(err, mirror));
+                    throw err;
+                }));
             });
             if (timeout !== undefined && timeout !== null) {
                 promises.push(sleep(timeout).then(() => 'timeout'));
             }
-            const res = await any(promises);
-            if (res === 'timeout') {
-                return undefined;
+            try {
+                const res = await any(promises);
+                if (res === 'timeout') {
+                    return undefined;
+                }
+                return res;
+            }
+            catch (err) {
+                this.logUnexpectedErrors('bulkQuery', errors);
+                throw err;
             }
-            return res;
+        }
+        async sequentialQuery(query, timeout) {
+            const startTime = Date.now();
+            const errors = [];
+            let lastError;
+            for (const mirror of this.mirrors) {
+                if (timeout !== undefined && timeout !== null) {
+                    const elapsed = Date.now() - startTime;
+                    if (elapsed >= timeout) {
+                        return undefined;
+                    }
+                }
+                try {
+                    const res = await this.post(mirror, query);
+                    return res?.data;
+                }
+                catch (err) {
+                    lastError = err;
+                    errors.push(this.extractErrorInfo(err, mirror));
+                    if (mirror !== this.mirrors[this.mirrors.length - 1]) {
+                        await sleep(100);
+                    }
+                }
+            }
+            this.logUnexpectedErrors('sequentialQuery', errors);
+            throw lastError;
         }
         reset() {
             this.mirrors = [OVERPASS_URL_ALT1, OVERPASS_URL_ALT2, OVERPASS_URL_ALT3];
             this.url = this.mirrors[0];
         }
+        getAppVersion() {
+            return this.getBindings()?.appInfo?.getAppVersion() ?? '';
+        }
+        getUserId() {
+            return this.getUserSettings().getValue('uuid', '');
+        }
+        getUserSettings() {
+            return useUserSettings();
+        }
+        getBindings() {
+            return getBindings();
+        }
     };
     return OverpassApi = _classThis;
 })();

package/lib/esm/src/services/overpass/overpass.js

@@ -110,6 +110,7 @@ let OverpassApi = (() => {
         }
         async query(queryOL, timeout) {
             const ts = Date.now();
+            console.log('# [overpass] query', queryOL);
             this.logger.logEvent({ message: 'query', queryOL, queryId: ts, timeout });
             const res = await this.bulkQuery(queryOL, timeout);
             this.logger.logEvent({ message: 'query result', queryId: ts, hasData: res !== undefined, duration: Date.now() - ts });
@@ -130,9 +131,11 @@ let OverpassApi = (() => {
         }
         async postWithErrorTracking(mirror, query) {
             try {
+                console.log('# [overpass] query', mirror);
                 return await this.postToOverpass(mirror, query);
             }
             catch (err) {
+                console.log('# [overpass] ERROR', mirror, err);
                 const status = err?.response?.status;
                 const statusText = err?.response?.statusText;
                 if (status && status !== 408 && status !== 504) {

package/lib/types/activities/base/utils/activity.d.ts

@@ -30,6 +30,7 @@ export declare class Activity implements ActivityInfo {
     markExporting(type: string, exporting: boolean): void;
     isExporting(type: string): boolean;
     export(type: string, observer?: Observer): Promise<boolean>;
+    protected getExportFileName(format: string): Promise<string>;
     upload(connectedApp: string, observer?: Observer): Promise<boolean>;
     markUploading(connectedApp: string, uploading: boolean): void;
     isUploading(connectedApp: string): boolean;

package/lib/types/services/overpass/overpass.d.ts

@@ -1,5 +1,7 @@
 import { AppApiBase } from '../../apps/base/api/base';
 import { EventLogger } from 'gd-eventlog';
+import { UserSettingsService } from '../../settings';
+import { AxiosResponse } from 'axios';
 export declare const OVERPASS_URL_DEFAULT = "https://overpass.kumi.systems/api/interpreter";
 export declare class OverpassApi extends AppApiBase {
     protected url: string;
@@ -8,11 +10,34 @@ export declare class OverpassApi extends AppApiBase {
     constructor(props?: {
         url: string;
     });
+    protected getOverpassHeaders(): {
+        'User-Agent': string;
+        Referer: string;
+    };
+    protected post(url: string, data?: object | string, conf?: object): Promise<AxiosResponse>;
+    protected postViaNodeHttps(https: any, url: string, data?: object | string): Promise<any>;
     query(queryOL: string, timeout?: number): Promise<JSON | string | undefined>;
     singleQuery(queryOL: string): Promise<JSON | string>;
     rotateUrl(): void;
     protected getBaseUrl(): string;
+    protected extractErrorInfo(err: any, mirror: string): {
+        mirror: string;
+        status?: number;
+        statusText?: string;
+        error: string;
+    };
+    protected logUnexpectedErrors(fn: string, errors: Array<{
+        mirror: string;
+        status?: number;
+        statusText?: string;
+        error: string;
+    }>): void;
     protected bulkQuery(query: string, timeout?: number): Promise<JSON | string | undefined>;
+    protected sequentialQuery(query: string, timeout?: number): Promise<JSON | string | undefined>;
     reset(): void;
+    protected getAppVersion(): string;
+    protected getUserId(): string;
+    protected getUserSettings(): UserSettingsService;
+    protected getBindings(): import("../../api").IncyclistBindings;
 }
 export declare const useOverpassApi: () => OverpassApi;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "incyclist-services",
-  "version": "1.7.66",
+  "version": "1.7.67",
   "peerDependencies": {
     "gd-eventlog": "^0.1.27"
   },

package/lib/cjs/api/form/index.js

@@ -1,2 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });