dequanto 0.0.1

package/src/json/JsonArrayStore.ts

@@ -0,0 +1,310 @@
+import { File } from 'atma-io'
+import { class_Dfr, Constructor } from 'atma-utils'
+import memd from 'memd'
+import alot from 'alot'
+
+import { JsonConvert } from 'class-json'
+import { Alot } from 'alot/alot'
+
+export interface IStoreOptions<T, TOut = T> {
+    path: string
+    key: (x: Partial<T>) => string | number
+    map? (x: T): TOut
+    Type?: Constructor<T>
+    trackHistory?: boolean
+    format?: boolean
+}
+
+export class JsonArrayStore<T> {
+    private array: T[];
+    private hash: { [key: string]: T }
+
+    private fs = new JsonArrayFs<T>(this.options.path, this.options.Type, this.options.map, this.options.format);
+
+    constructor (public options: IStoreOptions<T>) {
+        let keyFn = this.options.key;
+        if (keyFn == null) {
+            throw new Error('Key getter must be defined');
+        }
+    }
+
+    async query(): Promise<Alot<T>> {
+        let arr = await this.getAll();
+        return alot(arr);
+    }
+
+    async getAll(opts?: { cloned?: boolean }): Promise<T[]> {
+        if (this.fs.errored != null) {
+            throw this.fs.errored;
+        }
+        if (this.array == null) {
+            await this.restore();
+        }
+        let arr = this.array;
+        if (opts?.cloned) {
+            arr = JsonConvert.fromJSON(JsonConvert.toJSON(arr));
+        }
+        return arr;
+    }
+    async getDict (): Promise< { [key: string]: T } > {
+        if (this.fs.errored != null) {
+            throw this.fs.errored;
+        }
+        if (this.hash != null) {
+            return this.hash;
+        }
+        await this.restore();
+        return this.hash;
+    }
+
+    async getSingle (key: string | number) {
+        if (this.fs.errored != null) {
+            throw this.fs.errored;
+        }
+        if (this.hash == null) {
+            await this.restore();
+        }
+        return this.hash[ String(key) ];
+    }
+
+    async saveAll(arr: T[]): Promise<T[]> {
+        let keyFn = this.options.key;
+        this.array = arr;
+        this.hash = alot(arr).toDictionary(x => String(keyFn(x)), x => x);
+        await this.fs.write(arr);
+        return arr;
+    }
+
+    async upsert(x: Partial<T>) {
+        if (this.hash == null) {
+            await this.restore();
+        }
+        let entry = this.upsertSync(x);
+        await this.flush();
+        return entry;
+    }
+
+    async remove(key: string | number) {
+        if (this.hash == null) {
+            await this.restore();
+        }
+        let entry = this.removeSync(key);
+        await this.flush();
+        return entry;
+    }
+
+    async upsertMany(arr: Partial<T>[]): Promise<T[]> {
+        if (this.hash == null) {
+            await this.restore();
+        }
+        let entries = arr.map(x => this.upsertSync(x));
+        await this.flush();
+        return entries;
+    }
+
+
+    getLock () {
+        return this.fs.lock;
+    }
+
+    private async restore () {
+        let arr = await this.fs.read();
+        let keyFn = this.options.key;
+        if (this.array != null) {
+            // write was called inbetween
+            return;
+        }
+
+        this.array = arr;
+        this.hash = alot(arr).toDictionary(x => String(keyFn(x)), x => x);
+    }
+    private async flush () {
+        return await this.fs.write(this.array);
+    }
+
+    /** without flush */
+    private upsertSync(x: Partial<T>): T {
+        let keyFn = this.options.key;
+        let key = String(keyFn(x));
+        if (key == null) {
+            throw new Error(`Key is undefined in ${ JSON.stringify(x) }`);
+        }
+        let current = this.hash[key];
+        if (current) {
+            for (let prop in x) {
+                current[prop] = x[prop];
+            }
+            return current;
+        }
+
+        let entry = x as T;
+        let Type = this.options.Type;
+        if (Type != null && entry instanceof Type === false) {
+            entry = new Type(x);
+        }
+        this.array.push(entry);
+        this.hash[key] = entry;
+        return entry;
+    }
+    private removeSync(key: string | number): T {
+        let current = this.hash[key];
+        if (current == null) {
+            return null;
+        }
+
+        delete this.hash[key];
+        let i = this.array.indexOf(current);
+        if (i > -1) {
+            this.array.splice(i, 1);
+        }
+        return current;
+    }
+}
+
+class JsonArrayFs<T> {
+    public errored: Error = null;
+
+    private listeners = [] as ({version: number, promise: class_Dfr})[];
+    private version = 0;
+    private array: T[];
+    private pending: T[];
+    private busy = false;
+    private pathBak = this.path + '.bak';
+    private pathFilename = this.path.substring(this.path.lastIndexOf('/') + 1);
+
+    public lock = new class_Dfr;
+
+    constructor (public path: string, public Type?: Function,  public mapFn?: (x: T) => any, public format?: boolean) {
+        this.lock.resolve();
+    }
+
+    public write (arr: T[]) {
+        this.array = arr;
+
+        let dfr = new class_Dfr;
+        this.listeners.push({
+            version: ++this.version,
+            promise: dfr
+        });
+
+        if (this.busy === true) {
+            this.pending = arr;
+            return dfr;
+        }
+
+        this.busy = true;
+        this.lock.defer();
+        this.writeInner(arr);
+        return dfr;
+    }
+    public async read (): Promise<T[]> {
+        if (this.array != null) {
+            return Promise.resolve(this.array);
+        }
+        let arr = await this.readInner();
+        return this.array = arr;
+    }
+
+    @memd.deco.memoize({ perInstance: true })
+    private async readInner () {
+        let existsBak = await File.existsAsync (this.pathBak);
+        if (existsBak) {
+            let arr = await Fs.read(this.pathBak);
+            if (this.array) {
+                // When `write` was called inbetween `exists` check and now
+                return this.array;
+            }
+            if (arr) {
+                await File.renameAsync(this.pathBak, this.pathFilename);
+                return arr;
+            } else {
+                await File.removeAsync(this.pathBak);
+            }
+        }
+        let exists = await File.existsAsync (this.path);
+        if (exists === false) {
+            return [];
+        }
+        if (!this.path) {
+            throw new Error(`Read inner: ${this.path}/${this.pathBak} is undefined`);
+        }
+        let arr = await File.readAsync<T[]>(this.path);
+        if (this.Type) {
+            let rgx = Math.round(Math.random() * 100000);
+            let key = `JsonArray.Type ${rgx}`;
+            console.time(key);
+            arr = arr.map(x => JsonConvert.fromJSON(x, { Type: this.Type as any }));
+            console.timeEnd(key)
+        }
+        if (this.mapFn) {
+            arr = arr.map(this.mapFn);
+        }
+        return arr;
+    }
+
+    private async writeInner (arr: T[]) {
+        try {
+            let v = this.version;
+            let data = Fs.serialize(arr, this.Type, this.format);
+            await File.writeAsync(this.pathBak, data);
+            await File.renameAsync(this.pathBak, this.pathFilename);
+            this.callWriteListeners(v, null);
+        } catch (error) {
+            console.error(`JsonArrayStore.WriteInner`, error);
+            this.errored = error;
+        } finally {
+            if (this.pending == null) {
+                this.busy = false;
+                this.lock.resolve();
+                return;
+            }
+            let next = this.pending;
+            this.pending = null;
+            this.writeInner(next);
+        }
+    }
+
+    private callWriteListeners (v: number, error = null) {
+        for (let i = 0; i < this.listeners.length; i++) {
+            let x = this.listeners[i];
+            if (x.version <= v) {
+                try {
+                    if (error != null) {
+                        x.promise.reject(error);
+                    } else {
+                        x.promise.resolve();
+                    }
+                } finally {
+                    this.listeners.splice(i, 1);
+                    i--;
+                }
+            }
+        }
+    }
+}
+
+namespace Fs {
+    export function serialize <T = any> (arr: T[], Type?: Function, format?: boolean) {
+        if (Type) {
+            arr = arr.map(x => JsonConvert.toJSON(x))
+        }
+        return JSON.stringify(arr, null, format ? '  ' : null);
+    }
+    export function parse (str: string) {
+        return JSON.parse(str);
+    }
+    export async function read (path: string) {
+        if (!path) {
+            throw new Error(`${path} is undefined`);
+        }
+        let str = await File.readAsync <string> (path, { skipHooks: true, encoding: 'utf8' });
+        if (str.length === 0) {
+            return null;
+        }
+        try {
+            return JSON.parse(str);
+        } catch (error) {
+            return null;
+        }
+    }
+}

package/src/json/JsonObjectStore.ts

@@ -0,0 +1,239 @@
+import memd from 'memd'
+import { File } from 'atma-io'
+import { class_Dfr, Constructor } from 'atma-utils'
+import { JsonConvert } from 'class-json'
+import { $csv } from '@dequanto/utils/$csv';
+
+export interface IStoreOptions<T, TOut = T> {
+    path: string
+    map? (x: T): TOut
+    Type?: Constructor<T>
+    format?: boolean
+    default?: TOut
+}
+
+export class JsonObjectStore<T> {
+
+    private data: T;
+
+
+    private fs = new JsonFs<T>(
+        this.options.path,
+        this.options.Type,
+        this.options.map,
+        this.options.format
+    );
+
+    constructor (public options: IStoreOptions<T>) {
+
+    }
+
+
+    async get(opts?: { cloned?: boolean }): Promise<T> {
+        if (this.fs.errored != null) {
+            throw this.fs.errored;
+        }
+        if (this.data == null) {
+            await this.restore();
+        }
+        let arr = this.data;
+        if (opts?.cloned) {
+            arr = JsonConvert.fromJSON(JsonConvert.toJSON(arr));
+        }
+        return arr;
+    }
+
+
+    async save(arr: T): Promise<T> {
+        this.data = arr;
+        await this.fs.write(arr);
+        return arr;
+    }
+
+
+
+    getLock () {
+        return this.fs.lock;
+    }
+
+    private async restore () {
+        let data = await this.fs.read(this.options.default);
+        if (this.data != null) {
+            // write was called inbetween
+            return;
+        }
+        this.data = data;
+    }
+    private async flush () {
+        return await this.fs.write(this.data);
+    }
+
+}
+
+class JsonFs<T> {
+    public errored: Error = null;
+
+    private listeners = [] as ({version: number, promise: class_Dfr})[];
+    private version = 0;
+    private value: T;
+    private pending: T;
+    private busy = false;
+    // private pathBak = this.path + '.bak';
+    // private pathFilename = this.path.substring(this.path.lastIndexOf('/') + 1);
+
+    public lock = new class_Dfr;
+
+    private _file = new File(this.path, { threadSafe: true });
+
+    constructor (public path: string, public Type?: any,  public mapFn?: (x: T) => any, public format?: boolean) {
+
+    }
+
+    public write (value: T) {
+        this.value = value;
+
+        let dfr = new class_Dfr;
+        this.listeners.push({
+            version: ++this.version,
+            promise: dfr
+        });
+
+        if (this.busy === true) {
+            this.pending = value;
+            return dfr;
+        }
+
+        this.busy = true;
+        this.lock.defer();
+        this.writeInner(value);
+        return dfr;
+    }
+    public async read ($default): Promise<T> {
+        if (this.value != null) {
+            return Promise.resolve(this.value);
+        }
+        let arr = await this.readInner();
+        return this.value = arr ?? $default ?? {};
+    }
+
+    @memd.deco.memoize({ perInstance: true })
+    private async readInner () {
+        if (await this._file.existsAsync() === false) {
+            return null;
+        }
+        let str =  await this._file.readAsync<string>({ skipHooks: true });
+        let data = JsonUtil.fromString(
+            str,
+            {},
+            this.Type,
+            this._file.uri.toLocalFile(),
+        );
+
+        if (this.mapFn) {
+            data = Array.isArray(data)
+                ? data.map(this.mapFn)
+                : this.mapFn(data);
+        }
+        return data;
+    }
+
+    private async writeInner (value: T) {
+        try {
+            let v = this.version;
+            let str = JsonUtil.toString(
+                value,
+                this.Type,
+                this.format,
+                this._file.uri.toLocalFile(),
+            );
+            await this._file.writeAsync(str);
+            this.callWriteListeners(v, null);
+        } catch (error) {
+            console.error(`JsonArrayStore.WriteInner`, error);
+            this.errored = error;
+        } finally {
+            if (this.pending == null) {
+                this.busy = false;
+                this.lock.resolve();
+                return;
+            }
+            let next = this.pending;
+            this.pending = null;
+            this.writeInner(next);
+        }
+    }
+
+    private callWriteListeners (v: number, error = null) {
+        for (let i = 0; i < this.listeners.length; i++) {
+            let x = this.listeners[i];
+            if (x.version <= v) {
+                try {
+                    if (error != null) {
+                        x.promise.reject(error);
+                    } else {
+                        x.promise.resolve();
+                    }
+                } finally {
+                    this.listeners.splice(i, 1);
+                    i--;
+                }
+            }
+        }
+    }
+}
+
+namespace JsonUtil {
+    export function mapToJson (data: any | any[], Type?) {
+        if (Type == null) {
+            return data;
+        }
+        return Array.isArray(data)
+            ? data.map(x => JsonConvert.toJSON(x, { Type }))
+            : JsonConvert.toJSON(data, { Type });
+    }
+    export function mapFromJson (json: any | any[], Type?) {
+        if (Type == null) {
+            return json;
+        }
+        return Array.isArray(json)
+            ? json.map(x => JsonConvert.fromJSON(x, { Type }))
+            : JsonConvert.fromJSON(json, { Type });
+    }
+
+    export function toString (data: any | any[], Type, format: boolean, path: string) {
+        if (data == null) {
+            return null;
+        }
+        let json = mapToJson(data, Type);
+        return path.endsWith('.csv')
+            ? CSV.stringify(json)
+            : JSON.stringify(json, null, format ? '  ' : null);
+    }
+    export function fromString (str: string, $default, Type, path: string) {
+        if (str == null) {
+            if (Type) {
+                return JsonConvert.fromJSON($default, { Type });
+            }
+            return $default;
+        }
+        try {
+            let json = path.endsWith('.csv')
+                ? CSV.parse(str)
+                : JSON.parse(str);
+            return mapFromJson(json, Type);
+        } catch (err) {
+            return $default;
+        }
+    }
+}
+
+namespace CSV {
+    export function parse (str) {
+        return $csv.parseToRows(str)
+    }
+    export function stringify (rows) {
+        return $csv.stringify({
+            rows: rows
+        });
+    }
+}

package/src/json/JsonStore.spec.ts

@@ -0,0 +1,98 @@
+import { File } from 'atma-io'
+import { JsonArrayStore } from './JsonArrayStore';
+import { JsonObjectStore } from './JsonObjectStore';
+
+
+let pathArr = '/test/tmp/jsonstore-arr.json';
+let pathObj = '/test/tmp/jsonstore-obj.json';
+
+UTest({
+    'array': {
+        $before () {
+            File.exists(pathArr) && File.remove(pathArr);
+        },
+        async 'should handle multiple' () {
+            let store = new JsonArrayStore<any>({ path: pathArr, key: x => x.id });
+            let tick = Date.now();
+
+            store.upsert({ id: 1, letter: 'A', tick });
+            store.upsert({ id: 1, letter: 'B' });
+            store.upsert({ id: 1, letter: 'C' });
+            store.upsert({ id: 1, letter: 'D' });
+            await store.upsert({ id: 1, letter: 'E' });
+
+            let arr = await store.getAll();
+            eq_(arr.length, 1);
+            eq_(arr[0].letter, 'E');
+            eq_(arr[0].tick, tick);
+
+            store.upsert({ id: 1, letter: 'F' });
+            await store.upsert({ id: 1, letter: 'G' });
+
+            arr = await store.getAll();
+            eq_(arr.length, 1);
+            eq_(arr[0].letter, 'G');
+
+            await store.getLock();
+
+            let existsBak = await File.existsAsync(pathArr + '.bak');
+            eq_(existsBak, false);
+
+            let str = await File.readAsync <string> (pathArr, { skipHooks: true, cache: false });
+            eq_(str[0], '[');
+
+            let json = JSON.parse(str);
+            deepEq_(json, arr);
+        },
+        'should load file and then write': {
+            async $before () {
+                await File.writeAsync(pathArr, [
+                    { id: 'foo', letter: 'F'},
+                    { id: 'bar', letter: 'B'}
+                ]);
+                File.clearCache();
+            },
+            async 'load and upsert' () {
+                let store = new JsonArrayStore<any>({
+                    path: pathArr,
+                    key: x => x.id,
+                    map: (x) => {
+                        x.letter = x.letter.toLowerCase();
+                        return x;
+                    }
+                });
+                let all = await store.getAll();
+                let tick = Date.now();
+
+                eq_(all.length, 2);
+
+                all[0].tick = tick;
+                all[1].tick = tick;
+                await store.upsertMany(all);
+
+                File.clearCache();
+                let arr = await File.readAsync<any[]>(pathArr);
+                eq_(arr.length, 2);
+                eq_(arr[0].tick, tick);
+                eq_(arr[0].letter, 'f');
+            }
+        }
+    },
+    'object': {
+        $before () {
+            File.exists(pathObj) && File.remove(pathObj);
+        },
+        async 'should handle multiple' () {
+            let store = new JsonObjectStore<any>({ path: pathObj });
+            let tick = Date.now();
+
+            await store.save({  tick });
+
+            let obj = await store.get();
+            eq_(obj.tick, tick);
+
+            let obj1 = await File.readAsync <any> (pathObj, { cache: false });
+            eq_(obj1.tick, tick);
+        },
+    }
+})

package/src/loggers/ILogger.ts

@@ -0,0 +1,3 @@
+export interface ILogger {
+    write(row: any[]): this
+}

package/src/loggers/LoggerService.ts

@@ -0,0 +1,39 @@
+import { Everlog } from 'everlog';
+import { ILogger } from './ILogger';
+
+export class LoggerService implements ILogger {
+
+
+    private channel = Everlog.createChannel(this.name, {
+        directory: './logs/dequanto/',
+        columns: [
+            { type: 'date', name: 'Date' },
+            { type: 'string', name: 'Level' },
+            { type: 'string', name: 'Message' },
+        ]
+    })
+
+    constructor(public name: string = 'journal') {
+
+    }
+
+    warn (message: string) {
+        this.write([ new Date(), 'Warn', message ]);
+    }
+    error (message: string) {
+        this.write([ new Date(), 'Error', message ]);
+    }
+    info (message: string) {
+        this.write([ new Date(), 'Info', message ]);
+    }
+
+    write (row: string)
+    write (row: any[])
+    write (row: any[] | string) {
+        let arr = typeof row === 'string' ? [ row ] : row;
+        arr.unshift(new Date());
+        this.channel.writeRow(arr);
+        console.log(Array.isArray(row) ? row.join(' ') : row);
+        return this;
+    }
+}