nesoi 3.0.0 → 3.0.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (186) hide show
  1. package/README.md +10 -0
  2. package/lib/adapters/postgres/src/migrator/bucket.d.ts +20 -0
  3. package/lib/adapters/postgres/src/migrator/bucket.js +184 -0
  4. package/lib/adapters/postgres/src/migrator/csv.d.ts +7 -0
  5. package/lib/adapters/postgres/src/migrator/csv.js +72 -0
  6. package/lib/adapters/postgres/src/migrator/migration.d.ts +2 -18
  7. package/lib/adapters/postgres/src/migrator/migration.js +10 -158
  8. package/lib/adapters/postgres/src/migrator/migrator.js +8 -5
  9. package/lib/adapters/postgres/src/migrator/runner.d.ts +16 -6
  10. package/lib/adapters/postgres/src/migrator/runner.js +103 -34
  11. package/lib/adapters/postgres/src/postgres.bucket_adapter.d.ts +19 -22
  12. package/lib/adapters/postgres/src/postgres.bucket_adapter.js +116 -100
  13. package/lib/adapters/postgres/src/postgres.cli.d.ts +23 -3
  14. package/lib/adapters/postgres/src/postgres.cli.js +70 -10
  15. package/lib/adapters/postgres/src/postgres.config.d.ts +5 -0
  16. package/lib/adapters/postgres/src/postgres.config.js +2 -0
  17. package/lib/adapters/postgres/src/postgres.nql.d.ts +7 -3
  18. package/lib/adapters/postgres/src/postgres.nql.js +86 -32
  19. package/lib/adapters/postgres/src/postgres.provider.d.ts +18 -0
  20. package/lib/adapters/postgres/src/postgres.provider.js +77 -0
  21. package/lib/adapters/postgres/test/postgres.bucket_adapter.test.js +76 -39
  22. package/lib/compiler/apps/monolyth/monolyth_compiler.d.ts +3 -0
  23. package/lib/compiler/apps/monolyth/monolyth_compiler.js +24 -0
  24. package/lib/compiler/apps/monolyth/stages/2_build_typescript_stage.js +2 -1
  25. package/lib/compiler/apps/monolyth/stages/5_dump_cli_stage.js +1 -1
  26. package/lib/compiler/apps/monolyth/stages/6_dump_package_json_stage.js +1 -1
  27. package/lib/compiler/elements/bucket.element.js +26 -11
  28. package/lib/compiler/elements/constants.element.js +1 -1
  29. package/lib/compiler/elements/element.d.ts +2 -0
  30. package/lib/compiler/elements/message.element.js +4 -4
  31. package/lib/compiler/helpers/dump_helpers.js +5 -2
  32. package/lib/compiler/stages/7_dump_stage.js +2 -0
  33. package/lib/compiler/treeshake.js +9 -37
  34. package/lib/compiler/typescript/bridge/extract.js +12 -0
  35. package/lib/compiler/typescript/bridge/inject.js +3 -0
  36. package/lib/compiler/typescript/bridge/organize.js +3 -3
  37. package/lib/elements/blocks/block.builder.js +4 -2
  38. package/lib/elements/blocks/job/internal/resource_job.builder.d.ts +22 -20
  39. package/lib/elements/blocks/job/internal/resource_job.d.ts +2 -1
  40. package/lib/elements/blocks/job/internal/resource_job.js +17 -4
  41. package/lib/elements/blocks/job/job.js +3 -0
  42. package/lib/elements/blocks/job/job.types.d.ts +7 -0
  43. package/lib/elements/blocks/job/job.types.js +2 -0
  44. package/lib/elements/blocks/machine/machine.js +3 -2
  45. package/lib/elements/blocks/resource/resource.builder.js +2 -4
  46. package/lib/elements/blocks/resource/resource.d.ts +5 -3
  47. package/lib/elements/blocks/resource/resource.js +26 -17
  48. package/lib/elements/edge/controller/adapters/controller_adapter.d.ts +2 -1
  49. package/lib/elements/edge/controller/adapters/controller_adapter.js +11 -2
  50. package/lib/elements/edge/controller/controller.builder.d.ts +4 -5
  51. package/lib/elements/edge/controller/controller.builder.js +7 -7
  52. package/lib/elements/edge/controller/controller.d.ts +2 -1
  53. package/lib/elements/edge/controller/controller.js +8 -6
  54. package/lib/elements/entities/bucket/adapters/bucket_adapter.d.ts +61 -23
  55. package/lib/elements/entities/bucket/adapters/bucket_adapter.js +22 -13
  56. package/lib/elements/entities/bucket/adapters/memory.bucket_adapter.d.ts +21 -22
  57. package/lib/elements/entities/bucket/adapters/memory.bucket_adapter.js +68 -2
  58. package/lib/elements/entities/bucket/adapters/memory.nql.d.ts +10 -6
  59. package/lib/elements/entities/bucket/adapters/memory.nql.js +38 -3
  60. package/lib/elements/entities/bucket/adapters/slow_memory.bucket_adapter.d.ts +0 -20
  61. package/lib/elements/entities/bucket/adapters/slow_memory.bucket_adapter.js +46 -30
  62. package/lib/elements/entities/bucket/bucket.builder.d.ts +8 -2
  63. package/lib/elements/entities/bucket/bucket.builder.js +13 -19
  64. package/lib/elements/entities/bucket/bucket.config.d.ts +5 -1
  65. package/lib/elements/entities/bucket/bucket.d.ts +180 -19
  66. package/lib/elements/entities/bucket/bucket.js +662 -48
  67. package/lib/elements/entities/bucket/bucket.schema.d.ts +7 -1
  68. package/lib/elements/entities/bucket/bucket.schema.js +2 -1
  69. package/lib/elements/entities/bucket/bucket.types.d.ts +2 -7
  70. package/lib/elements/entities/bucket/cache/bucket_cache.d.ts +6 -2
  71. package/lib/elements/entities/bucket/cache/bucket_cache.js +12 -12
  72. package/lib/elements/entities/bucket/graph/bucket_graph.d.ts +32 -5
  73. package/lib/elements/entities/bucket/graph/bucket_graph.js +80 -111
  74. package/lib/elements/entities/bucket/graph/bucket_graph.schema.d.ts +3 -6
  75. package/lib/elements/entities/bucket/graph/bucket_graph.schema.js +1 -4
  76. package/lib/elements/entities/bucket/graph/bucket_graph_link.builder.d.ts +3 -7
  77. package/lib/elements/entities/bucket/graph/bucket_graph_link.builder.js +6 -2
  78. package/lib/elements/entities/bucket/model/bucket_model.builder.js +1 -1
  79. package/lib/elements/entities/bucket/model/bucket_model.convert.js +3 -3
  80. package/lib/elements/entities/bucket/model/bucket_model.schema.d.ts +37 -8
  81. package/lib/elements/entities/bucket/model/bucket_model.schema.js +25 -4
  82. package/lib/elements/entities/bucket/model/bucket_model_field.builder.d.ts +33 -14
  83. package/lib/elements/entities/bucket/model/bucket_model_field.builder.js +56 -13
  84. package/lib/elements/entities/bucket/query/nql.schema.d.ts +1 -0
  85. package/lib/elements/entities/bucket/query/nql_compiler.js +13 -2
  86. package/lib/elements/entities/bucket/query/nql_engine.d.ts +11 -4
  87. package/lib/elements/entities/bucket/query/nql_engine.js +20 -11
  88. package/lib/elements/entities/bucket/view/bucket_view.js +63 -35
  89. package/lib/elements/entities/bucket/view/bucket_view.schema.d.ts +5 -2
  90. package/lib/elements/entities/bucket/view/bucket_view_field.builder.d.ts +6 -2
  91. package/lib/elements/entities/bucket/view/bucket_view_field.builder.js +22 -16
  92. package/lib/elements/entities/constants/constants.schema.d.ts +1 -1
  93. package/lib/elements/entities/drive/drive_adapter.d.ts +44 -0
  94. package/lib/elements/entities/drive/drive_adapter.js +10 -0
  95. package/lib/elements/entities/drive/local.drive_adapter.d.ts +10 -0
  96. package/lib/elements/entities/drive/local.drive_adapter.js +34 -0
  97. package/lib/elements/entities/message/message.schema.d.ts +1 -0
  98. package/lib/elements/entities/message/message.schema.js +33 -0
  99. package/lib/elements/entities/message/message_parser.d.ts +5 -1
  100. package/lib/elements/entities/message/message_parser.js +56 -35
  101. package/lib/elements/entities/message/template/message_template.schema.d.ts +10 -8
  102. package/lib/elements/entities/message/template/message_template_field.builder.d.ts +16 -6
  103. package/lib/elements/entities/message/template/message_template_field.builder.js +25 -0
  104. package/lib/elements/entities/message/template/message_template_parser.js +2 -1
  105. package/lib/engine/apps/app.config.d.ts +32 -11
  106. package/lib/engine/apps/app.config.js +12 -0
  107. package/lib/engine/apps/app.d.ts +2 -0
  108. package/lib/engine/apps/app.js +3 -0
  109. package/lib/engine/apps/inline.app.d.ts +5 -3
  110. package/lib/engine/apps/inline.app.js +27 -12
  111. package/lib/engine/apps/monolyth/monolyth.app.d.ts +4 -2
  112. package/lib/engine/apps/monolyth/monolyth.app.js +22 -10
  113. package/lib/engine/auth/authn.d.ts +5 -1
  114. package/lib/engine/auth/zero.authn_provider.d.ts +4 -2
  115. package/lib/engine/auth/zero.authn_provider.js +2 -2
  116. package/lib/engine/cli/cli.d.ts +3 -1
  117. package/lib/engine/cli/cli.js +22 -3
  118. package/lib/engine/cli/cli_adapter.d.ts +2 -1
  119. package/lib/engine/cli/cli_adapter.js +2 -1
  120. package/lib/engine/cli/cli_input.d.ts +19 -0
  121. package/lib/engine/cli/cli_input.js +207 -0
  122. package/lib/engine/cli/ui.d.ts +1 -1
  123. package/lib/engine/cli/ui.js +2 -2
  124. package/lib/engine/daemon.d.ts +3 -2
  125. package/lib/engine/daemon.js +14 -2
  126. package/lib/engine/data/date.js +2 -2
  127. package/lib/engine/data/datetime.d.ts +40 -4
  128. package/lib/engine/data/datetime.js +70 -11
  129. package/lib/engine/data/decimal.d.ts +1 -1
  130. package/lib/engine/data/decimal.js +3 -3
  131. package/lib/engine/data/error.d.ts +21 -4
  132. package/lib/engine/data/error.js +23 -7
  133. package/lib/engine/data/file.d.ts +38 -0
  134. package/lib/engine/data/file.js +54 -0
  135. package/lib/engine/data/json.d.ts +6 -0
  136. package/lib/engine/data/json.js +26 -0
  137. package/lib/engine/data/obj.d.ts +1 -1
  138. package/lib/engine/data/trash.d.ts +14 -0
  139. package/lib/engine/data/trash.js +2 -0
  140. package/lib/engine/data/tree.d.ts +7 -12
  141. package/lib/engine/data/tree.js +101 -49
  142. package/lib/engine/module.d.ts +2 -1
  143. package/lib/engine/module.js +2 -5
  144. package/lib/engine/space.d.ts +1 -0
  145. package/lib/engine/space.js +6 -0
  146. package/lib/engine/transaction/nodes/bucket.trx_node.d.ts +184 -24
  147. package/lib/engine/transaction/nodes/bucket.trx_node.js +346 -451
  148. package/lib/engine/transaction/nodes/bucket_query.trx_node.d.ts +4 -2
  149. package/lib/engine/transaction/nodes/bucket_query.trx_node.js +27 -15
  150. package/lib/engine/transaction/nodes/job.trx_node.d.ts +2 -1
  151. package/lib/engine/transaction/nodes/job.trx_node.js +6 -0
  152. package/lib/engine/transaction/trx.d.ts +5 -2
  153. package/lib/engine/transaction/trx.js +2 -2
  154. package/lib/engine/transaction/trx_engine.config.d.ts +1 -3
  155. package/lib/engine/transaction/trx_engine.d.ts +2 -2
  156. package/lib/engine/transaction/trx_engine.js +14 -11
  157. package/lib/engine/transaction/trx_node.d.ts +14 -4
  158. package/lib/engine/transaction/trx_node.js +50 -8
  159. package/lib/engine/tree.d.ts +1 -1
  160. package/lib/engine/util/crypto.d.ts +50 -0
  161. package/lib/engine/util/crypto.js +89 -0
  162. package/lib/engine/util/deep.d.ts +5 -0
  163. package/lib/engine/util/deep.js +46 -0
  164. package/lib/engine/util/dotenv.d.ts +2 -8
  165. package/lib/engine/util/dotenv.js +14 -36
  166. package/lib/engine/util/hash.d.ts +3 -0
  167. package/lib/engine/util/hash.js +23 -0
  168. package/lib/engine/util/log.js +1 -1
  169. package/lib/engine/util/mime.d.ts +10 -0
  170. package/lib/engine/util/mime.js +389 -0
  171. package/lib/engine/util/parse.d.ts +6 -5
  172. package/lib/engine/util/parse.js +16 -15
  173. package/lib/engine/util/path.d.ts +3 -0
  174. package/lib/engine/util/path.js +92 -0
  175. package/lib/engine/util/rules.d.ts +4 -0
  176. package/lib/engine/util/rules.js +12 -0
  177. package/package.json +2 -2
  178. package/tools/compile.js +2 -2
  179. package/tools/dotenv.d.ts +1 -0
  180. package/tools/dotenv.js +4 -0
  181. package/tools/joaquin/job.d.ts +5 -5
  182. package/tools/joaquin/mock.d.ts +23 -2
  183. package/tools/joaquin/mock.js +127 -21
  184. package/tsconfig.build.tsbuildinfo +1 -1
  185. package/lib/adapters/postgres/test/postgres.bucket_query.test.d.ts +0 -0
  186. package/lib/adapters/postgres/test/postgres.bucket_query.test.js +0 -136
package/README.md ADDED
@@ -0,0 +1,10 @@
1
+ # Nesoi
2
+ Zero-dependency Typescript framework for data-driven applications.
3
+
4
+ ![Logo](/docs/header.png)
5
+
6
+ > :warning: This is a work-in-progress. Breaking changes happen frequently. Once it's production-ready this notice will be removed.
7
+
8
+ ## Getting Started
9
+
10
+ Check out our [Hands-On Guide](/docs/HANDS-ON.md) for more details.
@@ -0,0 +1,20 @@
1
+ import { $Bucket, $Space } from "../../../../elements";
2
+ import postgres from 'postgres';
3
+ import { AnyDaemon } from "../../../../engine/daemon";
4
+ import { BucketAdapterConfig } from "../../../../elements/entities/bucket/adapters/bucket_adapter";
5
+ import { Migration } from './migration';
6
+ export declare class BucketMigrator<S extends $Space, D extends AnyDaemon, ModuleName extends NoInfer<keyof S['modules']>> {
7
+ private daemon;
8
+ private sql;
9
+ private module;
10
+ private bucket;
11
+ private tableName;
12
+ protected schema: $Bucket;
13
+ protected config?: BucketAdapterConfig;
14
+ constructor(daemon: D, sql: postgres.Sql<any>, module: ModuleName, bucket: NoInfer<keyof S['modules'][ModuleName]['buckets']>, tableName: string);
15
+ generate(): Promise<Migration | undefined>;
16
+ private getCurrentSchema;
17
+ private generateFieldOptions;
18
+ private generateField;
19
+ private fieldType;
20
+ }
@@ -0,0 +1,184 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.BucketMigrator = void 0;
4
+ const daemon_1 = require("../../../../engine/daemon");
5
+ const migration_1 = require("./migration");
6
+ class BucketMigrator {
7
+ constructor(daemon, sql, module, bucket, tableName) {
8
+ this.daemon = daemon;
9
+ this.sql = sql;
10
+ this.module = module;
11
+ this.bucket = bucket;
12
+ this.tableName = tableName;
13
+ const daemonBucket = daemon_1.Daemon.getModule(daemon, module)
14
+ .buckets[bucket];
15
+ this.schema = daemonBucket.schema;
16
+ this.config = daemonBucket.adapter.config;
17
+ }
18
+ async generate() {
19
+ const current = await this.getCurrentSchema();
20
+ const options = this.generateFieldOptions(current);
21
+ if (!Object.keys(options).length) {
22
+ return;
23
+ }
24
+ const type = current ? 'alter' : 'create';
25
+ return new migration_1.Migration(this.module, type, this.tableName, options);
26
+ }
27
+ async getCurrentSchema() {
28
+ const columns = await this.sql `
29
+ SELECT column_name, data_type, is_nullable
30
+ FROM information_schema.columns
31
+ WHERE table_name = ${this.tableName}`;
32
+ if (!columns.length) {
33
+ return;
34
+ }
35
+ return columns.map(col => ({
36
+ ...col,
37
+ nullable: col.is_nullable === 'YES',
38
+ field_exists: false
39
+ }));
40
+ }
41
+ generateFieldOptions(current) {
42
+ let mappedCurrent;
43
+ if (current) {
44
+ mappedCurrent = {};
45
+ current.forEach(col => {
46
+ mappedCurrent[col.column_name] = col;
47
+ });
48
+ Object.keys(this.schema.model.fields)
49
+ .forEach(name => {
50
+ if (mappedCurrent[name]) {
51
+ mappedCurrent[name].field_exists = true;
52
+ }
53
+ });
54
+ }
55
+ const fields = {};
56
+ Object.entries(this.schema.model.fields)
57
+ .forEach(([name, field]) => {
58
+ const options = this.generateField(name, field, mappedCurrent);
59
+ if (options.length) {
60
+ fields[name] = options;
61
+ }
62
+ });
63
+ // Add meta fields when creating table
64
+ if (!current) {
65
+ const created_by = this.config?.meta.created_by || 'created_by';
66
+ fields[created_by] = [
67
+ new migration_1.MigrationField(undefined, created_by, {
68
+ create: { type: 'character(64)', nullable: true }
69
+ })
70
+ ];
71
+ const created_at = this.config?.meta.created_at || 'created_at';
72
+ fields[created_at] = [
73
+ new migration_1.MigrationField(undefined, created_at, {
74
+ create: { type: 'timestamp without time zone' }
75
+ })
76
+ ];
77
+ const updated_by = this.config?.meta.updated_by || 'updated_by';
78
+ fields[updated_by] = [
79
+ new migration_1.MigrationField(undefined, updated_by, {
80
+ create: { type: 'character(64)', nullable: true }
81
+ })
82
+ ];
83
+ const updated_at = this.config?.meta.updated_at || 'updated_at';
84
+ fields[updated_at] = [
85
+ new migration_1.MigrationField(undefined, updated_at, {
86
+ create: { type: 'timestamp without time zone' }
87
+ })
88
+ ];
89
+ }
90
+ return fields;
91
+ }
92
+ generateField(name, $, columns) {
93
+ const type = this.fieldType($);
94
+ const pk = $.name === 'id';
95
+ const nullable = !$.required;
96
+ // Table doesn't exist yet, only option is to create the field
97
+ if (!columns) {
98
+ return [new migration_1.MigrationField(undefined, $.name, {
99
+ create: { type, pk, nullable }
100
+ })];
101
+ }
102
+ // Table exists, evaluate options
103
+ else {
104
+ // Field exists, alter only what changed
105
+ const column = columns[$.name];
106
+ if (column) {
107
+ if ($.name === 'id') {
108
+ // Id can't be modified for now.
109
+ return [];
110
+ }
111
+ // TODO: check details such as
112
+ // - changes in decimal precision
113
+ // - changes in maxLength
114
+ // - changes in fk (!!!)
115
+ const _type = type.startsWith(column.data_type) ? undefined : column.data_type;
116
+ const _nullable = column.nullable !== nullable ? column.nullable : undefined;
117
+ if (_type !== undefined || _nullable !== undefined) {
118
+ return [new migration_1.MigrationField(undefined, $.name, {
119
+ alter: { type: _type, nullable: _nullable }
120
+ })];
121
+ }
122
+ return [];
123
+ }
124
+ // If field doesn't exists in columns, it might:
125
+ // - be a new field
126
+ // - be a field of the same type being renamed
127
+ else {
128
+ const options = [];
129
+ options.push(new migration_1.MigrationField(undefined, $.name, {
130
+ create: { type, pk, nullable }
131
+ }));
132
+ const deletedColumnsOfSameType = Object.values(columns)
133
+ .filter(col => !col.field_exists)
134
+ .filter(col => type.startsWith(col.data_type));
135
+ if (deletedColumnsOfSameType.length) {
136
+ // TODO: check details such as
137
+ // - changes in decimal precision
138
+ // - changes in maxLength
139
+ // - changes in fk (!!!)
140
+ deletedColumnsOfSameType.forEach(col => {
141
+ const _type = type.startsWith(col.data_type) ? undefined : col.data_type;
142
+ const _nullable = col.nullable !== nullable ? col.nullable : undefined;
143
+ if (_type !== undefined || _nullable !== undefined) {
144
+ options.push(new migration_1.MigrationField(undefined, $.name, {
145
+ alter: { name: col.column_name, type: _type, nullable: _nullable }
146
+ }));
147
+ }
148
+ });
149
+ }
150
+ return options;
151
+ }
152
+ }
153
+ }
154
+ fieldType($) {
155
+ if ($.name === 'id') {
156
+ if ($.type === 'string') {
157
+ return 'character(64) PRIMARY KEY';
158
+ }
159
+ return 'serial4 PRIMARY KEY';
160
+ }
161
+ let type = {
162
+ 'boolean': () => 'boolean',
163
+ 'date': () => 'date',
164
+ 'datetime': () => 'timestamp without time zone',
165
+ 'decimal': () => 'numeric(18,9)', // TODO: read from schema (not stored yet)
166
+ 'dict': () => 'jsonb',
167
+ 'enum': () => 'character(64)', // TODO: read from schema maxLength
168
+ 'file': () => 'jsonb',
169
+ 'float': () => 'double precision',
170
+ 'int': () => 'integer',
171
+ 'obj': () => 'jsonb',
172
+ 'string': () => 'character varying', // TODO: char() if maxLength
173
+ 'unknown': () => { throw new Error('An unknown field shouldn\'t be stored on SQL'); },
174
+ }[$.type]();
175
+ if ($.array) {
176
+ type += '[]';
177
+ }
178
+ if ($.required) {
179
+ type += ' NOT NULL';
180
+ }
181
+ return type;
182
+ }
183
+ }
184
+ exports.BucketMigrator = BucketMigrator;
@@ -0,0 +1,7 @@
1
+ import postgres from 'postgres';
2
+ export declare class CSV {
3
+ /**
4
+ * Import a csv table into the database
5
+ */
6
+ static import(sql: postgres.Sql<any>, tableName: string, csvpath: string): Promise<void>;
7
+ }
@@ -0,0 +1,72 @@
1
+ "use strict";
2
+ var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
3
+ if (k2 === undefined) k2 = k;
4
+ var desc = Object.getOwnPropertyDescriptor(m, k);
5
+ if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
6
+ desc = { enumerable: true, get: function() { return m[k]; } };
7
+ }
8
+ Object.defineProperty(o, k2, desc);
9
+ }) : (function(o, m, k, k2) {
10
+ if (k2 === undefined) k2 = k;
11
+ o[k2] = m[k];
12
+ }));
13
+ var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
14
+ Object.defineProperty(o, "default", { enumerable: true, value: v });
15
+ }) : function(o, v) {
16
+ o["default"] = v;
17
+ });
18
+ var __importStar = (this && this.__importStar) || (function () {
19
+ var ownKeys = function(o) {
20
+ ownKeys = Object.getOwnPropertyNames || function (o) {
21
+ var ar = [];
22
+ for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
23
+ return ar;
24
+ };
25
+ return ownKeys(o);
26
+ };
27
+ return function (mod) {
28
+ if (mod && mod.__esModule) return mod;
29
+ var result = {};
30
+ if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
31
+ __setModuleDefault(result, mod);
32
+ return result;
33
+ };
34
+ })();
35
+ Object.defineProperty(exports, "__esModule", { value: true });
36
+ exports.CSV = void 0;
37
+ const fs = __importStar(require("fs"));
38
+ const log_1 = require("../../../../engine/util/log");
39
+ class CSV {
40
+ /**
41
+ * Import a csv table into the database
42
+ */
43
+ static async import(sql, tableName, csvpath) {
44
+ log_1.Log.info('csv', 'import', `Importing objects from csv file '${csvpath}' into table '${tableName}'`);
45
+ const lines = fs.readFileSync(csvpath).toString().split('\n');
46
+ const keys = lines[0]
47
+ .split(',')
48
+ .map(v => v.match(/"(.*)"/)?.[1] || v);
49
+ const objs = lines.slice(1).map(line => {
50
+ const rows = line
51
+ .split(',')
52
+ .map(v => v.match(/"(.*)"/)?.[1] || v);
53
+ const obj = {};
54
+ for (let i = 0; i < keys.length; i++) {
55
+ obj[keys[i]] = rows[i];
56
+ }
57
+ return obj;
58
+ });
59
+ sql.begin(async (sql) => {
60
+ for (const obj of objs) {
61
+ log_1.Log.info('csv', 'import', `Inserting object ${obj.id}`, obj);
62
+ await sql `
63
+ INSERT INTO ${sql(tableName)}
64
+ ${sql(obj, keys)}
65
+ `;
66
+ }
67
+ }).catch(e => {
68
+ log_1.Log.error('csv', 'import', 'CSV Import failed, rolling back changes', e);
69
+ });
70
+ }
71
+ }
72
+ exports.CSV = CSV;
@@ -1,6 +1,3 @@
1
- import { $Bucket, $Space } from "../../../../elements";
2
- import postgres from 'postgres';
3
- import { AnyDaemon } from "../../../../engine/daemon";
4
1
  export type MigrationFieldOperation = {
5
2
  create: {
6
3
  type: string;
@@ -29,13 +26,14 @@ export declare class MigrationField {
29
26
  sql(): string | undefined;
30
27
  }
31
28
  export declare class Migration {
29
+ module: string;
32
30
  private type;
33
31
  private tableName;
34
32
  private options;
35
33
  name: string;
36
34
  private fields;
37
35
  private needsReview;
38
- constructor(type: 'create' | 'alter', tableName: string, options: Record<string, MigrationField[]>);
36
+ constructor(module: string, type: 'create' | 'alter', tableName: string, options: Record<string, MigrationField[]>);
39
37
  describe(): string;
40
38
  sqlUp(): string;
41
39
  sqlDown(): string;
@@ -57,17 +55,3 @@ export declare class Migration {
57
55
  private fnUp;
58
56
  private fnDown;
59
57
  }
60
- export declare class BucketMigrator<S extends $Space, D extends AnyDaemon, ModuleName extends NoInfer<keyof S['modules']>> {
61
- private daemon;
62
- private sql;
63
- private module;
64
- private bucket;
65
- private tableName;
66
- protected schema: $Bucket;
67
- constructor(daemon: D, sql: postgres.Sql<any>, module: ModuleName, bucket: NoInfer<keyof S['modules'][ModuleName]['buckets']>, tableName: string);
68
- generate(): Promise<Migration | undefined>;
69
- private getCurrentSchema;
70
- private generateFieldOptions;
71
- private generateField;
72
- private fieldType;
73
- }
@@ -33,13 +33,12 @@ var __importStar = (this && this.__importStar) || (function () {
33
33
  };
34
34
  })();
35
35
  Object.defineProperty(exports, "__esModule", { value: true });
36
- exports.BucketMigrator = exports.Migration = exports.MigrationField = void 0;
36
+ exports.Migration = exports.MigrationField = void 0;
37
37
  const string_1 = require("../../../../engine/util/string");
38
38
  const path = __importStar(require("path"));
39
39
  const fs = __importStar(require("fs"));
40
40
  const crypto_1 = require("crypto");
41
41
  const datetime_1 = require("../../../../engine/data/datetime");
42
- const daemon_1 = require("../../../../engine/daemon");
43
42
  class MigrationField {
44
43
  constructor(oldColumn, newColumn, operation) {
45
44
  this.oldColumn = oldColumn;
@@ -70,13 +69,14 @@ class MigrationField {
70
69
  }
71
70
  sql() {
72
71
  if ('create' in this.operation) {
73
- return `${this.newColumn} ${this.operation.create.type}`;
72
+ return `"${this.newColumn}" ${this.operation.create.type}`;
74
73
  }
75
74
  }
76
75
  }
77
76
  exports.MigrationField = MigrationField;
78
77
  class Migration {
79
- constructor(type, tableName, options) {
78
+ constructor(module, type, tableName, options) {
79
+ this.module = module;
80
80
  this.type = type;
81
81
  this.tableName = tableName;
82
82
  this.options = options;
@@ -87,6 +87,7 @@ class Migration {
87
87
  describe() {
88
88
  let str = '';
89
89
  str += '┌\n';
90
+ str += `│ ${(0, string_1.colored)('module: ' + this.module, 'darkgray')}\n`;
90
91
  str += `│ ${(0, string_1.colored)(this.name, 'lightcyan')}\n`;
91
92
  str += `│ ${this.needsReview
92
93
  ? (0, string_1.colored)('⚠ More than one option for some fields. Requires review.', 'red')
@@ -124,7 +125,9 @@ class Migration {
124
125
  return '';
125
126
  }
126
127
  save(dirpath = './migrations') {
127
- const filepath = path.join(dirpath, this.name + '.ts');
128
+ const filedir = path.join('modules', this.module, dirpath);
129
+ fs.mkdirSync(filedir, { recursive: true });
130
+ const filepath = path.join(filedir, this.name + '.ts');
128
131
  const { encoded, hash } = this.encode();
129
132
  let str = '';
130
133
  str += 'import { Migrator } from \'nesoi/lib/adapters/postgres/src/migrator\';\n';
@@ -180,7 +183,7 @@ class Migration {
180
183
  }
181
184
  fnUp() {
182
185
  let str = '';
183
- str += 'up: async (daemon, sql) => {\n';
186
+ str += 'up: async ({ sql }) => {\n';
184
187
  str += '\tawait sql`\n';
185
188
  str += '\t\t' + this.sqlUp().replace(/\n/g, '\n\t\t') + '\n';
186
189
  str += '\t`\n';
@@ -189,7 +192,7 @@ class Migration {
189
192
  }
190
193
  fnDown() {
191
194
  let str = '';
192
- str += 'down: async (daemon, sql) => {\n';
195
+ str += 'down: async ({ sql }) => {\n';
193
196
  str += '\tawait sql`\n';
194
197
  str += '\t\t' + this.sqlDown().replace(/\n/g, '\n\t\t') + '\n';
195
198
  str += '\t`\n';
@@ -198,154 +201,3 @@ class Migration {
198
201
  }
199
202
  }
200
203
  exports.Migration = Migration;
201
- class BucketMigrator {
202
- constructor(daemon, sql, module, bucket, tableName) {
203
- this.daemon = daemon;
204
- this.sql = sql;
205
- this.module = module;
206
- this.bucket = bucket;
207
- this.tableName = tableName;
208
- this.schema = daemon_1.Daemon.getModule(daemon, module)
209
- .buckets[bucket]
210
- .schema;
211
- }
212
- async generate() {
213
- const current = await this.getCurrentSchema();
214
- const options = this.generateFieldOptions(current);
215
- if (!Object.keys(options).length) {
216
- return;
217
- }
218
- const type = current ? 'alter' : 'create';
219
- return new Migration(type, this.tableName, options);
220
- }
221
- async getCurrentSchema() {
222
- const columns = await this.sql `
223
- SELECT column_name, data_type, is_nullable
224
- FROM information_schema.columns
225
- WHERE table_name = ${this.tableName}`;
226
- if (!columns.length) {
227
- return;
228
- }
229
- return columns.map(col => ({
230
- ...col,
231
- nullable: col.is_nullable === 'YES',
232
- field_exists: false
233
- }));
234
- }
235
- generateFieldOptions(current) {
236
- let mappedCurrent;
237
- if (current) {
238
- mappedCurrent = {};
239
- current.forEach(col => {
240
- mappedCurrent[col.column_name] = col;
241
- });
242
- Object.keys(this.schema.model.fields)
243
- .forEach(name => {
244
- if (mappedCurrent[name]) {
245
- mappedCurrent[name].field_exists = true;
246
- }
247
- });
248
- }
249
- const fields = {};
250
- Object.entries(this.schema.model.fields)
251
- .forEach(([name, field]) => {
252
- const options = this.generateField(name, field, mappedCurrent);
253
- if (options.length) {
254
- fields[name] = options;
255
- }
256
- });
257
- return fields;
258
- }
259
- generateField(name, $, columns) {
260
- const type = this.fieldType($);
261
- const pk = $.name === 'id';
262
- const nullable = !$.required;
263
- // Table doesn't exist yet, only option is to create the field
264
- if (!columns) {
265
- return [new MigrationField(undefined, $.name, {
266
- create: { type, pk, nullable }
267
- })];
268
- }
269
- // Table exists, evaluate options
270
- else {
271
- // Field exists, alter only what changed
272
- const column = columns[$.name];
273
- if (column) {
274
- if ($.name === 'id') {
275
- // Id can't be modified for now.
276
- return [];
277
- }
278
- // TODO: check details such as
279
- // - changes in decimal precision
280
- // - changes in maxLength
281
- // - changes in fk (!!!)
282
- const _type = type.startsWith(column.data_type) ? undefined : column.data_type;
283
- const _nullable = column.nullable !== nullable ? column.nullable : undefined;
284
- if (_type !== undefined || _nullable !== undefined) {
285
- return [new MigrationField(undefined, $.name, {
286
- alter: { type: _type, nullable: _nullable }
287
- })];
288
- }
289
- return [];
290
- }
291
- // If field doesn't exists in columns, it might:
292
- // - be a new field
293
- // - be a field of the same type being renamed
294
- else {
295
- const options = [];
296
- options.push(new MigrationField(undefined, $.name, {
297
- create: { type, pk, nullable }
298
- }));
299
- const deletedColumnsOfSameType = Object.values(columns)
300
- .filter(col => !col.field_exists)
301
- .filter(col => type.startsWith(col.data_type));
302
- if (deletedColumnsOfSameType.length) {
303
- // TODO: check details such as
304
- // - changes in decimal precision
305
- // - changes in maxLength
306
- // - changes in fk (!!!)
307
- deletedColumnsOfSameType.forEach(col => {
308
- const _type = type.startsWith(col.data_type) ? undefined : col.data_type;
309
- const _nullable = col.nullable !== nullable ? col.nullable : undefined;
310
- if (_type !== undefined || _nullable !== undefined) {
311
- options.push(new MigrationField(undefined, $.name, {
312
- alter: { name: col.column_name, type: _type, nullable: _nullable }
313
- }));
314
- }
315
- });
316
- }
317
- return options;
318
- }
319
- }
320
- }
321
- fieldType($) {
322
- if ($.name === 'id') {
323
- if ($.type === 'string') {
324
- return 'character(64) PRIMARY KEY';
325
- }
326
- return 'serial4 PRIMARY KEY';
327
- }
328
- let type = {
329
- 'boolean': () => 'boolean',
330
- 'date': () => 'date',
331
- 'datetime': () => 'timestamp without time zone',
332
- 'decimal': () => 'numeric(18,9)', // TODO: read from schema (not stored yet)
333
- 'dict': () => 'jsonb',
334
- 'enum': () => 'character(64)', // TODO: read from schema maxLength
335
- 'file': () => { throw new Error('A file field shouldn\'t be stored on SQL'); },
336
- 'float': () => 'double precision',
337
- 'int': () => 'integer',
338
- 'obj': () => 'jsonb',
339
- 'string': () => 'character varying', // TODO: char() if maxLength
340
- 'unknown': () => { throw new Error('An unknown field shouldn\'t be stored on SQL'); },
341
- }[$.type]();
342
- if ($.array) {
343
- type += '[]';
344
- }
345
- if ($.required) {
346
- type += ' NOT NULL';
347
- }
348
- return type;
349
- }
350
- }
351
- exports.BucketMigrator = BucketMigrator;
@@ -2,11 +2,10 @@
2
2
  Object.defineProperty(exports, "__esModule", { value: true });
3
3
  exports.Migrator = void 0;
4
4
  const log_1 = require("../../../../engine/util/log");
5
- const migration_1 = require("./migration");
6
5
  const runner_1 = require("./runner");
7
6
  const daemon_1 = require("../../../../engine/daemon");
8
- const bucket_1 = require("../../../../elements/entities/bucket/bucket");
9
7
  const string_1 = require("../../../../engine/util/string");
8
+ const bucket_1 = require("./bucket");
10
9
  class Migrator {
11
10
  constructor(daemon, sql, dirpath = './migrations') {
12
11
  this.daemon = daemon;
@@ -21,6 +20,7 @@ class Migrator {
21
20
  if (!oldTable.length) {
22
21
  await migrator.sql `CREATE TABLE ${migrator.sql(Migrator.MIGRATION_TABLE_NAME)} (
23
22
  id SERIAL PRIMARY KEY,
23
+ module VARCHAR NOT NULL,
24
24
  name VARCHAR NOT NULL,
25
25
  description VARCHAR,
26
26
  batch INT4 NOT NULL,
@@ -29,7 +29,7 @@ class Migrator {
29
29
  filehash VARCHAR NOT NULL
30
30
  )`;
31
31
  }
32
- migrator.status = await runner_1.MigrationRunner.status(migrator.sql, migrator.dirpath);
32
+ migrator.status = await runner_1.MigrationRunner.status(daemon, migrator.sql, migrator.dirpath);
33
33
  return migrator;
34
34
  }
35
35
  async generate() {
@@ -38,7 +38,10 @@ class Migrator {
38
38
  for (const module of modules) {
39
39
  const buckets = daemon_1.Daemon.getModule(this.daemon, module.name).buckets;
40
40
  for (const bucket in buckets) {
41
- const adapter = bucket_1.Bucket.getAdapter(buckets[bucket]);
41
+ const schema = buckets[bucket].schema;
42
+ if (schema.module !== module.name)
43
+ continue;
44
+ const adapter = buckets[bucket].adapter;
42
45
  if (!adapter?.tableName)
43
46
  continue;
44
47
  const migration = await this.generateForBucket(module.name, bucket, adapter.tableName);
@@ -50,7 +53,7 @@ class Migrator {
50
53
  return migrations;
51
54
  }
52
55
  async generateForBucket(module, bucket, tableName) {
53
- const migrator = new migration_1.BucketMigrator(this.daemon, this.sql, module, bucket, tableName);
56
+ const migrator = new bucket_1.BucketMigrator(this.daemon, this.sql, module, bucket, tableName);
54
57
  const migration = await migrator.generate();
55
58
  const tag = (0, string_1.colored)(`${module}::bucket:${bucket}`, 'lightcyan');
56
59
  if (!migration) {
@@ -1,9 +1,14 @@
1
1
  import postgres from 'postgres';
2
2
  import { Migration } from './migration';
3
- import { AnyDaemon, Daemon } from "../../../../engine/daemon";
4
- type MigrationFn = (daemon: Daemon<any, any>, sql: postgres.Sql<any>) => Promise<void>;
3
+ import { AnyDaemon } from "../../../../engine/daemon";
4
+ import { AnyTrxNode } from "../../../../engine/transaction/trx_node";
5
+ type MigrationFn = ($: {
6
+ sql: postgres.Sql<any>;
7
+ trx: AnyTrxNode;
8
+ }) => Promise<void>;
5
9
  export type MigrationRow = {
6
10
  id: number;
11
+ module: string;
7
12
  name: string;
8
13
  description?: string;
9
14
  batch: number;
@@ -25,6 +30,7 @@ export declare class MigrationStatus {
25
30
  items: {
26
31
  state: 'done' | 'pending' | 'lost';
27
32
  id?: number;
33
+ module: string;
28
34
  name: string;
29
35
  description?: string;
30
36
  batch?: number;
@@ -33,8 +39,9 @@ export declare class MigrationStatus {
33
39
  filehash?: string;
34
40
  method?: MigrationMethod;
35
41
  }[];
36
- nextBatch: number;
42
+ batch: number;
37
43
  constructor(fileMigrations: {
44
+ module: string;
38
45
  name: string;
39
46
  hash: string;
40
47
  path: string;
@@ -46,16 +53,19 @@ export declare class MigrationRunner {
46
53
  dirpath: string;
47
54
  private migrations;
48
55
  private constructor();
49
- static scanFiles(dirpath: string): Promise<{
56
+ static scanFiles(daemon: AnyDaemon, dirpath: string): Promise<{
57
+ module: string;
50
58
  name: string;
51
59
  path: string;
52
60
  hash: string;
53
61
  method: MigrationMethod;
54
62
  }[]>;
55
63
  static scanDb(sql: postgres.Sql<any>): Promise<postgres.RowList<MigrationRow[]>>;
56
- static status(sql: postgres.Sql<any>, dirpath: string): Promise<MigrationStatus>;
64
+ static status(daemon: AnyDaemon, sql: postgres.Sql<any>, dirpath: string): Promise<MigrationStatus>;
57
65
  static up(daemon: AnyDaemon, sql: postgres.Sql<any>, mode?: 'one' | 'batch', dirpath?: string): Promise<void>;
58
- static oneUp(daemon: AnyDaemon, sql: postgres.Sql<any>, migration: Migration, dirpath?: string): Promise<void>;
66
+ static down(daemon: AnyDaemon, sql: postgres.Sql<any>, mode?: 'one' | 'batch', dirpath?: string): Promise<void>;
67
+ static injectUp(daemon: AnyDaemon, sql: postgres.Sql<any>, migration: Migration, dirpath?: string): Promise<void>;
59
68
  private static migrateUp;
69
+ private static migrateDown;
60
70
  }
61
71
  export {};