@v1nt1248/3nclient-lib 0.0.13 → 0.0.15

package/src/libs/serialization-for-ipc/json-n-binary.ts

@@ -0,0 +1,275 @@
+/*
+ Copyright (C) 2022 - 2023 3NSoft Inc.
+ 
+ This program is free software: you can redistribute it and/or modify it under
+ the terms of the GNU General Public License as published by the Free Software
+ Foundation, either version 3 of the License, or (at your option) any later
+ version.
+ 
+ This program is distributed in the hope that it will be useful, but
+ WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+ See the GNU General Public License for more details.
+ 
+ You should have received a copy of the GNU General Public License along with
+ this program. If not, see <http://www.gnu.org/licenses/>.
+*/
+
+import root from './json-ipc.proto';
+import { ProtoType } from './protobuf-type';
+import { copyJSON } from './copy-json';
+
+interface ValuesSequence {
+	values: Value[];
+}
+
+interface Value {
+	json?: string;
+	binaryInJson?: BinaryValue[];
+	transferredInJson?: TransferredObj[];
+	arr?: BinaryValue;
+	transferred?: TransferredObj;
+}
+
+interface BinaryValue {
+	arr: Uint8Array;
+	objLocation: string[];
+}
+
+interface TransferredObj {
+	indexInPassed: number;
+	objLocation: string[];
+}
+
+const valuesType = ProtoType.for<ValuesSequence>(root.json_ipc.ValuesSequence);
+
+export function serializeArgs(
+	args: any[]
+): { bytes: Uint8Array; passedByReference?: any[]; } {
+	const { seq, passedByReference } = argsToValuesSequence(args);
+	return { bytes: valuesType.pack(seq), passedByReference };
+}
+
+function argsToValuesSequence(
+	args: any[]
+	): { seq: ValuesSequence; passedByReference?: any[]; } {
+	const seq: ValuesSequence = { values: [] };
+	const passedByReference: any[] = [];
+	for (const arg of args) {
+		if (arg && (typeof arg === 'object')) {
+			if (ArrayBuffer.isView(arg)) {
+				seq.values.push({
+					arr: { arr: arg as Uint8Array, objLocation: [] }
+				});
+			} else if ((arg as ObjectFromCore)._isObjectFromCore) {
+				const indexInPassed = addToArray(passedByReference, arg);
+				seq.values.push({
+					transferred: { indexInPassed, objLocation: [] }
+				});
+			} else {
+				seq.values.push(turnToJsonExtractingBinaryAndTransferable(
+					arg, passedByReference
+				));
+			}
+		} else {
+			seq.values.push({
+				json: JSON.stringify(arg)
+			});
+		}
+	}
+	return {
+		seq,
+		passedByReference: ((passedByReference.length > 0) ?
+			passedByReference : undefined
+		)
+	};
+}
+
+function addToArray(arr: Array<any>, o: any): number {
+	let foundIndex = arr.indexOf(o);
+	if (foundIndex < 0) {
+		arr.push(o);
+		return arr.length - 1;
+	} else {
+		return foundIndex;
+	}
+}
+
+function turnToJsonExtractingBinaryAndTransferable<T extends object>(
+	arg: T, passedByReference: any[]
+): {
+	json: string; binaryInJson?: BinaryValue[];
+	transferredInJson?: TransferredObj[];
+} {
+	const parts = extractNonJsonableFrom(arg, passedByReference);
+	if (parts) {
+		const { copy, binaryInJson, transferredInJson } = parts;
+		return {
+			json: JSON.stringify(copy),
+			binaryInJson: ((binaryInJson.length > 0) ? binaryInJson : undefined),
+			transferredInJson: ((transferredInJson.length > 0) ?
+				transferredInJson : undefined
+			)
+		};
+	} else {
+		return { json: JSON.stringify(arg) };
+	}
+}
+
+function extractNonJsonableFrom<T extends object>(
+	arg: T, passedByReference: any[]
+): {
+	copy: T; binaryInJson: BinaryValue[];
+	transferredInJson: TransferredObj[];
+}|undefined {
+	const nonJsonLocations = findAllNonJsonable(arg);
+	if (!nonJsonLocations) { return; }
+	const copy = copyJSON(arg);
+	const binaryInJson: BinaryValue[] = [];
+	const transferredInJson: TransferredObj[] = [];
+	for (const objLocation of nonJsonLocations) {
+		const nonJson = getValueAtObjLocation(arg, objLocation);
+		setNewValueAtObjLocation(copy, objLocation, null);
+		if ((nonJson as ObjectFromCore)._isObjectFromCore) {
+			const indexInPassed = addToArray(passedByReference, nonJson);
+			transferredInJson.push({ indexInPassed, objLocation });
+		} else {
+			binaryInJson.push({ arr: nonJson, objLocation });
+		}
+	}
+	return { copy, binaryInJson, transferredInJson };
+}
+
+interface ObjectFromCore {
+	_isObjectFromCore: true;
+}
+
+function findAllNonJsonable(o: object): string[][]|undefined {
+	const foundObjLocations: string[][] = [];
+	if (ArrayBuffer.isView(o)
+	|| (o as ObjectFromCore)._isObjectFromCore) {
+		return [ [] ];
+	}
+	if (Array.isArray(o)) {
+		for (let i=0; i<o.length; i+=1) {
+			const child = o[i];
+			if (child && (typeof child === 'object')) {
+				const inChild = findAllNonJsonable(child);
+				if (inChild) {
+					for (const objLocation of inChild) {
+						foundObjLocations.push([ `${i}`, ...objLocation ]);
+					}
+				}
+			}
+		}
+	} else {
+		for (const [ field, child ] of Object.entries(o)) {
+			if (child && (typeof child === 'object')) {
+				const inChild = findAllNonJsonable(child);
+				if (inChild) {
+					for (const objLocation of inChild) {
+						foundObjLocations.push([ field, ...objLocation ]);
+					}
+				}
+			}
+		}
+	}
+	return ((foundObjLocations.length > 0) ? foundObjLocations : undefined);
+}
+
+function getValueAtObjLocation(o: object, objLocation: string[]): any {
+	const value = (o as any)[objLocation[0]];
+	if (objLocation.length > 1) {
+		return getValueAtObjLocation(value, objLocation.slice(1));
+	} else {
+		return value;
+	}
+}
+
+function setNewValueAtObjLocation(
+	o: object, objLocation: string[], newValue: any
+): void {
+	const value = (o as any)[objLocation[0]];
+	if (objLocation.length > 1) {
+		setNewValueAtObjLocation(value, objLocation.slice(1), newValue);
+	} else {
+		(o as any)[objLocation[0]] = newValue;
+	}
+}
+
+function getInitAndSetNewValueAt(
+	o: object, objLocation: string[], newValue: any
+): any {
+	const value = (o as any)[objLocation[0]];
+	if (objLocation.length > 1) {
+		return getInitAndSetNewValueAt(value, objLocation.slice(1), newValue);
+	} else {
+		(o as any)[objLocation[0]] = newValue;
+		return value;
+	}
+}
+
+export function deserializeArgs(
+	bytes: Uint8Array, passedByReference: any[]|undefined
+): any[] {
+	const values = valuesType.unpack(bytes as Buffer);
+	const args: any[] = [];
+	for (const val of values.values) {
+		const {
+			json, binaryInJson, transferredInJson, arr, transferred
+		} = val;
+		if (arr) {
+			args.push(arr.arr);
+		} else if (transferred) {
+			args.push(getTransferred(
+				transferred.indexInPassed, passedByReference
+			));
+		} else if ((typeof json === 'string') && (json.length > 0)) {
+			const arg = JSON.parse(json);
+			if (binaryInJson) {
+				attachBinaryArrays(arg, binaryInJson);
+			}
+			if (transferredInJson) {
+				attachTransferred(arg, transferredInJson, passedByReference);
+			}
+			args.push(arg);
+		} else {
+			// XXX throw error here
+		}
+	}
+	return args;
+}
+
+function attachBinaryArrays<T extends object>(
+	arg: T, binaryInJson: BinaryValue[]
+): void {
+	for (const { arr, objLocation } of binaryInJson) {
+		setNewValueAtObjLocation(arg, objLocation, arr);
+	}
+}
+
+function getTransferred(
+	indexInPassed: number, passedByReference: any[]|undefined
+): any {
+	if (!passedByReference) {
+		// XXX throw error here
+		throw new Error(`need better error`);
+	}
+	const o = passedByReference[indexInPassed];
+	if (!o || !(o as ObjectFromCore)._isObjectFromCore) {
+		// XXX throw error here
+		throw new Error(`need better error`);
+	}
+	return o;
+}
+
+function attachTransferred<T extends object>(
+	arg: T, transferredInJson: TransferredObj[],
+	passedByReference: any[]|undefined
+): void {
+	for (const { indexInPassed, objLocation } of transferredInJson) {
+		setNewValueAtObjLocation(arg, objLocation, getTransferred(
+			indexInPassed, passedByReference
+		));
+	}
+}

package/src/libs/serialization-for-ipc/protobuf-type.ts

@@ -0,0 +1,61 @@
+/*
+ Copyright (C) 2020 - 2022 3NSoft Inc.
+ 
+ This program is free software: you can redistribute it and/or modify it under
+ the terms of the GNU General Public License as published by the Free Software
+ Foundation, either version 3 of the License, or (at your option) any later
+ version.
+ 
+ This program is distributed in the hope that it will be useful, but
+ WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+ See the GNU General Public License for more details.
+ 
+ You should have received a copy of the GNU General Public License along with
+ this program. If not, see <http://www.gnu.org/licenses/>.
+*/
+
+import { Type as PBType } from 'protobufjs';
+
+
+export class ProtoType<T extends object> {
+
+	private constructor(
+		private readonly type: PBType
+	) {
+		this.type = type;
+		Object.freeze(this);
+	}
+
+	static for<T extends object>(type: any): ProtoType<T> {
+		return new ProtoType<T>(type);
+	}
+
+	pack(msg: T): Buffer {
+		const err = this.type.verify(msg);
+		if (err) { throw new Error(err); }
+		return this.type.encode(msg).finish() as Buffer;
+	}
+
+	unpack(bytes: Buffer|void): T {
+		if (!bytes) {
+			throw {
+				runtimeException: true,
+				type: 'ipc',
+				missingBodyBytes: true
+			};
+		}
+		return this.type.decode(bytes) as T;
+	}
+
+	packToBase64(msg: T): string {
+		return this.pack(msg).toString('base64');
+	}
+
+	unpackFromBase64(str: string): T {
+		return this.unpack(Buffer.from(str, 'base64'));
+	}
+
+}
+Object.freeze(ProtoType.prototype);
+Object.freeze(ProtoType);

package/src/libs/sqlite-on-3nstorage/deferred.ts

@@ -0,0 +1,32 @@
+/*
+ Copyright (C) 2015, 2017, 2019 - 2022 3NSoft Inc.
+ 
+ This program is free software: you can redistribute it and/or modify it under
+ the terms of the GNU General Public License as published by the Free Software
+ Foundation, either version 3 of the License, or (at your option) any later
+ version.
+ 
+ This program is distributed in the hope that it will be useful, but
+ WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+ See the GNU General Public License for more details.
+ 
+ You should have received a copy of the GNU General Public License along with
+ this program. If not, see <http://www.gnu.org/licenses/>.
+*/
+
+export interface Deferred<T> {
+	promise: Promise<T>;
+	resolve: (result?: T|PromiseLike<T>) => void;
+	reject: (err: any) => void;
+}
+
+export function defer<T>(): Deferred<T> {
+	const d = <Deferred<T>> {};
+	d.promise = new Promise<T>((resolve, reject) => {
+		d.resolve = resolve as Deferred<T>['resolve'];
+		d.reject = reject;
+	});
+	Object.freeze(d);
+	return d;
+}

package/src/libs/sqlite-on-3nstorage/index.ts

@@ -0,0 +1,291 @@
+/*
+ Copyright (C) 2022 3NSoft Inc.
+ 
+ This program is free software: you can redistribute it and/or modify it under
+ the terms of the GNU General Public License as published by the Free Software
+ Foundation, either version 3 of the License, or (at your option) any later
+ version.
+ 
+ This program is distributed in the hope that it will be useful, but
+ WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+ See the GNU General Public License for more details.
+ 
+ You should have received a copy of the GNU General Public License along with
+ this program. If not, see <http://www.gnu.org/licenses/>.
+*/
+
+import initSqlJs, { Database as DBClass, BindParams as QueryParams, QueryExecResult as QueryResult } from './sqljs';
+import { SingleProc, Action } from './synced';
+
+export type Database = DBClass;
+export type BindParams = QueryParams;
+export type QueryExecResult = QueryResult;
+
+type WritableFile = web3n.files.WritableFile;
+type ReadonlyFile = web3n.files.ReadonlyFile;
+type FileException = web3n.files.FileException;
+
+export interface SaveOpts {
+	skipUpload?: boolean;
+}
+
+
+export abstract class SQLiteOn3NStorage {
+
+	protected readonly syncProc = new SingleProc();
+
+	protected constructor(
+		protected readonly database: Database,
+		protected readonly file: WritableFile
+	) {}
+
+	static async makeAndStart(file: WritableFile): Promise<SQLiteOn3NStorage> {
+		const SQL = await initSqlJs(true);
+		const fileContent = await readFileContent(file);
+		const db = new SQL.Database(fileContent);
+		let sqlite: SQLiteOn3NStorage;
+		if (file.v?.sync) {
+			sqlite = new SQLiteOnSyncedFS(db, file);
+		} else if (file.v) {
+			sqlite = new SQLiteOnLocalFS(db, file);
+		} else {
+			sqlite = new SQLiteOnDeviceFS(db, file);
+		}
+		await sqlite.start();
+		return sqlite;
+	}
+
+	private async start(): Promise<void> {
+		// XXX add listening process(es)
+
+	}
+
+	async saveToFile(opts?: SaveOpts): Promise<void> {
+		await this.syncProc.startOrChain(async () => {
+			const dbFileContent = this.database.export();
+			await this.file.writeBytes(dbFileContent);
+		});
+	}
+
+	get db(): Database {
+		return this.database;
+	}
+
+	sync<T>(action: Action<T>): Promise<T> {
+		return this.syncProc.startOrChain(action);
+	}
+
+	listTables(): string[] {
+		const result = this.database.exec(
+			`SELECT tbl_name FROM sqlite_schema WHERE type='table' AND name NOT LIKE 'sqlite_%'`
+		);
+		return ((result.length > 0) ?
+			result[0].values.map(row => row[0] as string) :
+			[]
+		);
+	}
+
+}
+Object.freeze(SQLiteOn3NStorage.prototype);
+Object.freeze(SQLiteOn3NStorage);
+
+
+class SQLiteOnSyncedFS extends SQLiteOn3NStorage {
+
+	constructor(db: Database, file: WritableFile) {
+		super(db, file);
+		Object.seal(this);
+	}
+
+	async saveToFile(opts?: SaveOpts): Promise<void> {
+		await super.saveToFile();
+		if (opts?.skipUpload) {
+			return;
+		} else {
+			await this.file.v!.sync!.upload();
+		}
+	}
+
+}
+Object.freeze(SQLiteOnSyncedFS.prototype);
+Object.freeze(SQLiteOnSyncedFS);
+
+
+class SQLiteOnLocalFS extends SQLiteOn3NStorage {
+
+	constructor(db: Database, file: WritableFile) {
+		super(db, file);
+		Object.seal(this);
+	}
+
+}
+Object.freeze(SQLiteOnLocalFS.prototype);
+Object.freeze(SQLiteOnLocalFS);
+
+
+class SQLiteOnDeviceFS extends SQLiteOn3NStorage {
+
+	constructor(db: Database, file: WritableFile) {
+		super(db, file);
+		Object.seal(this);
+	}
+
+}
+Object.freeze(SQLiteOnDeviceFS.prototype);
+Object.freeze(SQLiteOnDeviceFS);
+
+
+async function readFileContent(
+	file: ReadonlyFile
+): Promise<Uint8Array|undefined> {
+	try {
+		return await file.readBytes();
+	} catch (exc) {
+		if ((exc as FileException).notFound) {
+			return undefined;
+		} else {
+			throw exc;
+		}
+	}
+}
+
+export function objectFromQueryExecResult<T>(
+	sqlResult: QueryExecResult
+): Array<T> {
+	const { columns, values: rows } = sqlResult;
+	return rows.map(row => row.reduce((obj, cellValue, index) => {
+		const field = columns[index] as keyof T;
+		obj[field] = cellValue as any;
+		return obj;
+	}, {} as T));
+}
+
+
+export class TableColumnsAndParams<ColumnDefs extends object> {
+
+	public readonly c: { [columnName in keyof ColumnDefs]: string; };
+	public readonly cReversed: { [snakedColName: string ]: keyof ColumnDefs; };
+	public readonly p: { [columnName in keyof ColumnDefs]: string; };
+	public readonly q: { [columnName in keyof ColumnDefs]: string; };
+
+	constructor(
+		public readonly name: string,
+		private readonly columnDefs: ColumnDefs
+	) {
+		this.c = {} as this['c'];
+		this.cReversed = {} as this['cReversed'];
+		this.p = {} as this['p'];
+		this.q = {} as this['q'];
+		for (const cName of Object.keys(this.columnDefs)) {
+			const snakedColName = toSnakeCaseName(cName);
+			this.c[cName as keyof ColumnDefs] = snakedColName;
+			this.cReversed[snakedColName] = cName as keyof ColumnDefs;
+			this.p[cName as keyof ColumnDefs] = `$${cName}`;
+			this.q[cName as keyof ColumnDefs] = `${this.name}.${snakedColName}`;
+		}
+		Object.freeze(this.c);
+		Object.freeze(this.p);
+		Object.freeze(this.q);
+	}
+
+	private toC(cName: string): string {
+		const snakedColName = this.c[cName as keyof ColumnDefs];
+		if (snakedColName === undefined) {
+			throw new Error(`Column ${cName} is not found among columns of table ${this.name}`);
+		}
+		return snakedColName;
+	}
+
+	toParams<T extends { [columnName in keyof ColumnDefs]: any; }>(
+		value: Partial<T>, throwOnUnknownField = true
+	): any {
+		const params = {} as any;
+		for (const [cName, columnValue] of Object.entries(value)) {
+			this.toC(cName);	// does implicit check for column existence
+			params[this.p[cName as keyof ColumnDefs]] = columnValue;
+		}
+		for (const paramName of Object.values(this.p)) {
+			if (params[paramName as string] === undefined) {
+				params[paramName as string] = null;
+			}
+		}
+		return params;
+	}
+	
+	getFromQueryExecResult<T>(
+		sqlResult: QueryExecResult
+	): Array<T> {
+		const { columns, values: rows } = sqlResult;
+		return rows.map(row => row.reduce((obj, cellValue, index) => {
+			const tabColumn = columns[index];
+			let field = this.cReversed[tabColumn as string];
+			if (field === undefined) {
+				field = tabColumn as keyof ColumnDefs;
+			}
+			obj[field as string as keyof T] = cellValue as any;
+			return obj;
+		}, {} as T));
+	}
+
+	get insertQuery(): string {
+		const colAndParamNames = Object.entries(this.p);
+		return `INSERT INTO ${this.name} (${
+			colAndParamNames.map(([cName]) => this.toC(cName)).join(', ')
+		}) VALUES (${
+			colAndParamNames.map(([n, colParam]) => colParam).join(', ')
+		})`;
+	}
+
+	updateQuery(
+		withTabName: boolean,
+		columns: (keyof ColumnDefs)[]|undefined = undefined,
+		skipColumns = false
+	): string {
+		let colAndParamNames = Object.entries(this.p);
+		if (columns) {
+			if (skipColumns) {
+				colAndParamNames = colAndParamNames.filter(
+					([cName]) => !columns.includes(cName as keyof ColumnDefs)
+				);
+			} else {
+				colAndParamNames = colAndParamNames.filter(
+					([cName]) => columns.includes(cName as keyof ColumnDefs)
+				);
+			}
+		}
+		return `UPDATE ${withTabName ? `${this.name} ` : ''}SET ${
+			colAndParamNames
+			.map(([cName, pName]) => `${this.toC(cName)}=${pName}`)
+			.join(', ')
+		}`;
+	}
+
+	get columnsCreateSection(): string {
+		return Object.entries(this.columnDefs)
+		.map(([cName, columnDef]) => `${this.toC(cName)} ${columnDef}`)
+		.join(`,\n`);
+	}
+
+	selectQuery(
+		columnsToSelect: (keyof ColumnDefs)[] | string,
+		...whereAndColEqual: (keyof ColumnDefs)[]
+	): string {
+		const whereClause = (whereAndColEqual.length > 0) ?
+		` WHERE ${
+			(whereAndColEqual as string[]).map(n => `${n}=$${n}`).join(' AND ')
+		}` : '';
+		return `SELECT ${(typeof columnsToSelect === 'string') ?
+			this.toC(columnsToSelect) :
+			columnsToSelect.map(cName => this.toC(cName as string)).join(', ')
+		} FROM ${this.name}${whereClause}`;
+	}
+
+}
+Object.freeze(TableColumnsAndParams.prototype);
+Object.freeze(TableColumnsAndParams);
+
+
+function toSnakeCaseName(str: string): string {
+	return str.replace(/[A-Z]/g, (letter) => `_${letter.toLowerCase()}`);
+}