@gershy/lilac 0.0.14 → 0.0.16

package/cmp/cjs/util/aws.js

@@ -4,13 +4,13 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.regions = exports.capitalKeys = void 0;
-const clearing_1 = require("@gershy/clearing");
+require("@gershy/clearing");
 const util_phrasing_1 = __importDefault(require("@gershy/util-phrasing"));
 const capitalKeys = (v) => {
-    if ((0, clearing_1.isCls)(v, Array))
-        return v[map](v => (0, exports.capitalKeys)(v));
-    if ((0, clearing_1.isCls)(v, Object))
-        return v[mapk]((val, key) => [(0, util_phrasing_1.default)(key, 'camel', 'kamel'), (0, exports.capitalKeys)(val)]);
+    if (cl.isCls(v, Array))
+        return v[cl.map](v => (0, exports.capitalKeys)(v));
+    if (cl.isCls(v, Object))
+        return v[cl.mapk]((val, key) => [(0, util_phrasing_1.default)('camel->kamel', key), (0, exports.capitalKeys)(val)]);
     return v;
 };
 exports.capitalKeys = capitalKeys;
@@ -54,7 +54,7 @@ exports.regions = [
     // Ignoring these for now...
     // 'us-gov-east-1',
     // 'us-gov-west-1',
-][map](region => {
+][cl.map](region => {
     const [c, z, num] = region.split('-');
     const [dir0, dir1 = dir0] = z.match(/central|north|south|east|west/g);
     return {

package/cmp/cjs/util/normalize.d.ts

@@ -1,2 +1,3 @@
+import '@gershy/clearing';
 declare const normalize: (val: any, seen?: Set<any>) => any;
 export default normalize;

package/cmp/cjs/util/normalize.js

@@ -1,13 +1,14 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-const clearing_1 = require("@gershy/clearing");
+require("@gershy/clearing");
+const { isCls, getClsName } = clearing;
 const normalize = (val, seen = new Set()) => {
     // Derives a json-stringifiable value from *any* value
     // E.g. to hash *anything*: hash(JSON.stringify(normalized(anything)));
     // Handles terminations
-    if ((0, clearing_1.isCls)(val, String))
+    if (isCls(val, String))
         return val;
-    if ((0, clearing_1.isCls)(val, Number))
+    if (isCls(val, Number))
         return val;
     if (val === null)
         return null;
@@ -16,10 +17,10 @@ const normalize = (val, seen = new Set()) => {
     if (seen.has(val))
         return '<<!circ!>>';
     seen.add(val);
-    if ((0, clearing_1.isCls)(val, Array))
-        return val[map](v => normalize(v, seen));
-    if ((0, clearing_1.isCls)(val, Object))
+    if (isCls(val, Array))
+        return val[cl.map](v => normalize(v, seen));
+    if (isCls(val, Object))
         return normalize(Object.entries(val).sort((e0, e1) => e0[0] < e1[0] ? -1 : 1), seen);
-    return normalize({ $form: (0, clearing_1.getClsName)(val), ...val }, seen);
+    return normalize({ $form: getClsName(val), ...val }, seen);
 };
 exports.default = normalize;

package/cmp/cjs/util/procTerraform.d.ts

@@ -1,6 +1,8 @@
 import type { Fact } from '@gershy/disk';
 import { ProcOpts } from '@gershy/nodejs-proc';
-export type ProcTerraformArgs = ProcOpts & {};
+export type ProcTerraformArgs = ProcOpts & {
+    config?: string;
+};
 declare const _default: (fact: Fact, cmd: string, opts?: ProcTerraformArgs) => Promise<{
     logDb: Fact;
     output: string;

package/cmp/cjs/util/procTerraform.js

@@ -17,14 +17,19 @@ exports.default = (fact, cmd, opts = {}) => {
         timeoutMs: 0,
         ...opts,
         cwd: fact,
-        env: { TF_DATA_DIR: '' }
+        env: {
+            ...process.env,
+            TF_DATA_DIR: '',
+            TF_CLI_CONFIG_FILE: '',
+            ...opts.env
+        }
     });
     return Object.assign(prm.then(async (result) => {
         const logDb = await writeLog(result.output);
         return { logDb, output: result.output.split('\n').slice(-numTailingTfLogLines).join('\n') };
     }, async (err) => {
-        const logDb = await writeLog(err.output ?? err[limn]());
-        throw Error(`terraform failed (${err.message})`)[mod]({
+        const logDb = await writeLog(err.output ?? err[cl.limn]());
+        throw Error(`terraform failed (${err.message})`)[cl.mod]({
             logDb,
             ...(err.output ? { output: err.output.split('\n').slice(-numTailingTfLogLines).join('\n') } : { cause: err })
         });

package/cmp/cjs/util/slashEscape.js

@@ -1,7 +1,7 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.default = (str, escapeChars) => {
-    if (!escapeChars[has]('\\'))
+    if (!escapeChars[cl.has]('\\'))
         escapeChars += '\\';
     // The regex to construct must have "\" and "]" escaped:
     const escChars = escapeChars.replace(/([\\\]])/g, '\\$1');

package/cmp/cjs/util/terraform.js

@@ -6,7 +6,7 @@ const embed = (v) => '${' + v + '}';
 exports.embed = embed;
 const json = (v) => [
     '| <<EOF',
-    JSON.stringify(v, null, 2)[indent](2),
+    JSON.stringify(v, null, 2)[cl.indent](2),
     'EOF'
 ].join('\n');
 exports.json = json;

package/cmp/mjs/main.d.ts

@@ -1,7 +1,8 @@
 import '../sideEffects.js';
 import { PetalTerraform } from './petal/terraform/terraform.ts';
 import type Logger from '@gershy/logger';
-import type { Fact } from '@gershy/disk';
+import { type Fact } from '@gershy/disk';
+import '@gershy/clearing';
 import { Soil } from './soil/soil.ts';
 import { SuperIterable } from './util/superIterable.ts';
 export type Context = {
@@ -9,6 +10,7 @@ export type Context = {
     logger: Logger;
     fact: Fact;
     patioFact: Fact;
+    shedFact: Fact;
     maturity: string;
     debug: boolean;
     pfx: string;
@@ -18,6 +20,7 @@ export declare class Flower {
     constructor();
     getDependencies(): Generator<Flower>;
     getPetals(ctx: Context): SuperIterable<PetalTerraform.Base>;
+    cultivate(): Promise<void>;
 }
 type RegistryFlowers<R extends Registry<any>, M extends 'real' | 'test'> = R extends Registry<infer Flowers> ? {
     [K in keyof Flowers]: Flowers[K][M];
@@ -39,13 +42,14 @@ export declare class Garden<Reg extends Registry<any>> {
     private ctx;
     private reg;
     private def;
+    private tfProcArgs;
     constructor(args: {
         context: Context;
         registry: Reg;
         define: Garden<Reg>['def'];
     });
     private getPetals;
-    genTerraform(deployTarget: Soil.Base): Promise<Obj<Fact>>;
+    genTerraform(soil: Soil.Base): Promise<Obj<Fact>>;
     private terraformInit;
     private terraformPlan;
     private terraformApply;

package/cmp/mjs/main.js

@@ -2,15 +2,24 @@
 // A more generic (beyond just tf) provider is very hard to support due to the multiplicity of
 // provider/petal combos - e.g. "api" flower would need to support ,api.getCloudformationPetals,
 // api.getTerraformPetals, etc... supporting just terraform for now
-// Watch out when working lilac into an npm dependency, need to allow the user a way to declare
-// their absolute repo path (so "<repo>/..." filenames work in any setup!)
-// Can region be dealt with any better??
 // Support test-mode (Flowers need to be able to do setup, share config, write to volumes, etc)
 import { PetalTerraform } from "./petal/terraform/terraform.js";
-import { isCls, skip } from '@gershy/clearing';
-import procTerraform from "./util/procTerraform.js";
+import { tempFact } from '@gershy/disk';
+import '@gershy/clearing';
 import tryWithHealing from '@gershy/util-try-with-healing';
 import phrasing from '@gershy/util-phrasing';
+import proc from '@gershy/nodejs-proc';
+const { isCls, skip } = cl;
+const toArr = cl.toArr;
+const allObj = cl.allObj;
+const allArr = cl.allArr;
+const has = cl.has;
+const map = cl.map;
+const mod = cl.mod;
+const walk = cl.walk;
+const merge = cl.merge;
+const upper = cl.upper;
+const baseline = cl.baseline;
 export class Flower {
     // TODO: The downside of having this static is that different instances may use different
     // services - e.g. api gateway instance may have "useEdge: true", in which case we'd like to
@@ -27,6 +36,11 @@ export class Flower {
     getPetals(ctx) {
         throw Error('not implemented');
     }
+    async cultivate() {
+        // This function is called once all Flowers for a given Garden have been constructed. The main
+        // purpose of this phase is to allow Flowers which reference each other via functions to run
+        // such functions without running into uninitialized values.
+    }
 }
 ;
 export class Registry {
@@ -40,7 +54,7 @@ export class Registry {
     }
     getAwsServices() {
         const services = new Set();
-        for (const [k, { real }] of this.flowers)
+        for (const [k, { real }] of this.flowers[walk]())
             for (const awsService of real.getAwsServices())
                 services.add(awsService);
         return services[toArr](v => v);
@@ -58,11 +72,21 @@ export class Garden {
     ctx;
     reg;
     def;
+    tfProcArgs;
     constructor(args) {
         const { define, registry, context } = args;
         this.ctx = context;
         this.reg = registry;
         this.def = define;
+        this.tfProcArgs = {
+            timeoutMs: 0,
+            env: {
+                ...process.env,
+                TF_LOG: 'DEBUG',
+                TF_DATA_DIR: '',
+                TF_CLI_CONFIG_FILE: ''
+            }
+        };
     }
     async *getPetals() {
         // TODO: We always use the "real" flowers from the registry - this is part of the shift to
@@ -72,21 +96,22 @@ export class Garden {
         // conditionally calling `this.registry.get('fake')`...
         const seenFlowers = new Set();
         const seenPetals = new Set();
-        for await (const topLevelFlower of await this.def(this.ctx, this.reg.get('real'))) {
-            for (const flower of topLevelFlower.getDependencies()) {
-                if (seenFlowers.has(flower))
-                    continue;
+        for await (const topLevelFlower of await this.def(this.ctx, this.reg.get('real')))
+            for (const flower of topLevelFlower.getDependencies())
                 seenFlowers.add(flower);
-                for await (const petal of await flower.getPetals(this.ctx)) {
-                    if (seenPetals.has(petal))
-                        continue;
-                    yield petal;
-                }
+        // Now we've exhaustively referenced all Flowers - we can cultivate them
+        await Promise.all(seenFlowers[toArr](f => f.cultivate()));
+        for (const flower of seenFlowers) {
+            for await (const petal of await flower.getPetals(this.ctx)) {
+                if (seenPetals.has(petal))
+                    continue;
+                seenPetals.add(petal);
+                yield petal;
             }
         }
     }
-    async genTerraform(deployTarget) {
-        const soilTfPetalsPrm = deployTarget.getTerraformPetals(this.ctx);
+    async genTerraform(soil) {
+        const soilTfPetalsPrm = soil.getTerraformPetals(this.ctx);
         return this.ctx.logger.scope('garden.genTerraform', {}, async (logger) => {
             const setupTfProj = async (args) => args.logger.scope('tf', { proj: this.ctx.name, tf: args.term }, async (logger) => {
                 // Allows a terraform project to be defined in terms of a function which writes to main.tf,
@@ -108,8 +133,7 @@ export class Garden {
                     const stream = await args.fact.kid(['main.tf']).getDataHeadStream();
                     await args.setup(args.fact, stream, async (petal) => {
                         // Include a utility function the caller can use to easily write petals
-                        const { tf, files = {} } = await petal.getResult()
-                            .then(tf => isCls(tf, String) ? { tf } : tf);
+                        const { tf, files = {} } = await petal.getResult().then(tf => isCls(tf, String) ? { tf } : tf);
                         if (tf)
                             await stream.write(`${tf}\n`);
                         await Promise.all(files[toArr]((data, kfp) => args.fact.kid(kfp.split('/')).setData(data)));
@@ -154,7 +178,7 @@ export class Garden {
                         // Create ddb tf state locking table
                         await writePetalTfAndFiles(new PetalTerraform.Resource('awsDynamodbTable', 'tfState', {
                             name: ddbName,
-                            billingMode: phrasing('payPerRequest', 'camel', 'snake')[upper](),
+                            billingMode: phrasing('camel->snake', 'payPerRequest')[upper](),
                             hashKey: 'LockID',
                             $attribute: { name: 'LockID', type: 'S' }
                         }));
@@ -181,31 +205,55 @@ export class Garden {
             });
         });
     }
-    terraformInit(fact, args) {
+    // TODO: Write terraform output to logs??
+    async terraformInit(fact) {
+        // Consider if we ever want to pass "-reconfigure" and "-migrate-state" options; these are
+        // useful if we are moving backends (e.g. one aws account to another), and want to move our
+        // full iac definition too
+        // Ensure the mirror directory exists in the shed
+        const mirrorFact = this.ctx.shedFact.kid(['lilacTerraformMirror']);
+        await mirrorFact.kid(['note.txt']).setData(`Root of terraform mirror for @gershy/lilac`);
         return this.ctx.logger.scope('execTf.init', { fact: fact.fsp() }, async (logger) => {
-            // Consider if we ever want to pass "-reconfigure" and "-migrate-state" options; these are
-            // useful if we are moving backends (e.g. one aws account to another), and want to move our
-            // full iac definition too
-            // TODO: Some terraform commands fail when offline - can this be covered up? Possibly by
-            // checking terraform binaries into the repo? (Cross-platform nightmare though...)
-            const result = await procTerraform(fact, `terraform init -input=false`, {
-                onData: async (mode, data) => logger.log({ $$: 'notice', mode, data }) ?? null
+            const { output: result } = await tryWithHealing({
+                fn: () => logger.scope('attempt', {}, async (logger) => {
+                    const configFact = tempFact.kid([`${Math.random().toString(36).slice(2)}.terraform.rc`]);
+                    await configFact.setData(String[baseline](`
+            | provider_installation {
+            |   filesystem_mirror {
+            |     path = "${mirrorFact.fsp().replaceAll('\\', '/')}"
+            |   }
+            | }
+          `));
+                    return proc(`terraform init -input=false`, {}[merge](this.tfProcArgs)[merge]({
+                        cwd: fact,
+                        env: { TF_CLI_CONFIG_FILE: configFact.fsp() }
+                    })).finally(() => configFact.rem());
+                }),
+                canHeal: err => (err?.output ?? '')[has]('Could not retrieve the list of available versions for provider'),
+                heal: () => logger.scope('mirror', { fsp: mirrorFact.fsp() }, async (logger) => {
+                    const { output: result } = await proc(`terraform providers mirror "${mirrorFact.fsp().replaceAll('\\', '/')}"`, { cwd: fact, timeoutMs: 0 });
+                    logger.log({ $$: 'result', result });
+                })
             });
-            logger.log({ $$: 'result', logFp: result.logDb.toString(), msg: result.output });
+            logger.log({ $$: 'result', result });
             return result;
         });
     }
     terraformPlan(fact, args) {
         return this.ctx.logger.scope('execTf.plan', { fact: fact.fsp() }, async (logger) => {
-            const result = await procTerraform(fact, `terraform plan -input=false`);
-            logger.log({ $$: 'result', logFp: result.logDb.toString(), msg: result.output });
+            const { output: result } = await proc(`terraform plan -input=false`, {}[merge](this.tfProcArgs)[merge]({
+                cwd: fact,
+            }));
+            logger.log({ $$: 'result', result });
             return result;
         });
     }
     terraformApply(fact, args) {
         return this.ctx.logger.scope('execTf.apply', { fact: fact.fsp() }, async (logger) => {
-            const result = await procTerraform(fact, `terraform apply -input=false -auto-approve`);
-            logger.log({ $$: 'result', logFp: result.logDb.toString(), msg: result.output });
+            const { output: result } = await proc(`terraform apply -input=false -auto-approve`, {}[merge](this.tfProcArgs)[merge]({
+                cwd: fact
+            }));
+            logger.log({ $$: 'result', result });
             return result;
         });
     }

package/cmp/mjs/petal/terraform/terraform.d.ts

@@ -1,3 +1,4 @@
+import '@gershy/clearing';
 import Petal from '../petal.ts';
 export declare namespace PetalTerraform {
     class Base extends Petal {

package/cmp/mjs/petal/terraform/terraform.js

@@ -1,7 +1,15 @@
-import { getClsName, isCls } from '@gershy/clearing';
+import '@gershy/clearing';
 import slashEscape from "../../util/slashEscape.js";
 import Petal from "../petal.js";
 import ph from '@gershy/util-phrasing';
+const { getClsName, isCls } = cl;
+const hasHead = cl.hasHead;
+const map = cl.map;
+const hasTail = cl.hasTail;
+const indent = cl.indent;
+const mod = cl.mod;
+const toArr = cl.toArr;
+const has = cl.has;
 export var PetalTerraform;
 (function (PetalTerraform) {
     class Base extends Petal {
@@ -49,13 +57,13 @@ export var PetalTerraform;
                         throw Error('tf key of this form must correspond to object value')[mod]({ k, v });
                     // Resolve to raw string?
                     if (special && isCls(v, String))
-                        return [ph(pcs[0], 'camel', 'snake'), ' = ', this.terraformEncode(v[hasHead]('| ') ? v : `| ${v}`)];
+                        return [ph('camel->snake', pcs[0]), ' = ', this.terraformEncode(v[hasHead]('| ') ? v : `| ${v}`)];
                     // Resolve to nested block?
                     if (special && isCls(v, Object))
-                        return [[ph(pcs[0], 'camel', 'snake'), ...pcs.slice(1)[map](pc => ph(pc, 'camel', 'snake'))].join(' '), ' ', this.terraformEncode(v)];
+                        return [[ph('camel->snake', pcs[0]), ...pcs.slice(1)[map](pc => ph('camel->snake', pc))].join(' '), ' ', this.terraformEncode(v)];
                     // Resolve anything else to typical property - use the key exactly as provided (to support,
                     // e.g., aws format for keys in policies, any other specific format, etc.)
-                    return [ph(pcs[0], 'camel', 'snake'), ' = ', this.terraformEncode(v)];
+                    return [ph('camel->snake', pcs[0]), ' = ', this.terraformEncode(v)];
                 });
                 const len = entryItems.length;
                 if (len === 0)
@@ -76,9 +84,9 @@ export var PetalTerraform;
         refStr(props = []) {
             if (!isCls(props, Array))
                 props = [props];
-            const base = `${ph(this.getType(), 'camel', 'snake')}.${ph(this.getHandle(), 'camel', 'snake')}`;
+            const base = `${ph('camel->snake', this.getType())}.${ph('camel->snake', this.getHandle())}`;
             return props.length
-                ? `${base}.${props[map](v => ph(v, 'camel', 'snake')).join('.')}`
+                ? `${base}.${props[map](v => ph('camel->snake', v)).join('.')}`
                 : base;
         }
         ref(props = []) {
@@ -124,7 +132,7 @@ export var PetalTerraform;
         getHandle() { return this.handle; }
         getProps() { return this.props; }
         async getResultHeader() {
-            return `resource "${ph(this.type, 'camel', 'snake')}" "${ph(this.handle, 'camel', 'snake')}"`;
+            return `resource "${ph('camel->snake', this.type)}" "${ph('camel->snake', this.handle)}"`;
         }
     }
     PetalTerraform.Resource = Resource;
@@ -139,7 +147,7 @@ export var PetalTerraform;
         }
         getProps() { return this.props; }
         async getResultHeader() {
-            return `provider "${ph(this.name, 'camel', 'snake')}"`;
+            return `provider "${ph('camel->snake', this.name)}"`;
         }
     }
     PetalTerraform.Provider = Provider;
@@ -158,7 +166,7 @@ export var PetalTerraform;
         getHandle() { return this.handle; }
         getProps() { return this.props; }
         async getResultHeader() {
-            return `data "${ph(this.type, 'camel', 'snake')}" "${ph(this.handle, 'camel', 'snake')}"`;
+            return `data "${ph('camel->snake', this.type)}" "${ph('camel->snake', this.handle)}"`;
         }
         refStr(props = []) {
             return `data.${super.refStr(props)}`;

package/cmp/mjs/soil/soil.d.ts

@@ -1,8 +1,10 @@
 import { Context, PetalTerraform, Registry } from '../main.ts';
 import { RegionTerm } from '../util/aws.ts';
+import '@gershy/clearing';
 import { NetProc } from '@gershy/util-http';
 import { SuperIterable } from '../util/superIterable.ts';
 import type Logger from '@gershy/logger';
+import { RestApi } from '@aws-sdk/client-api-gateway';
 export declare namespace Soil {
     type PetalProjArgs = {
         s3Name: string;
@@ -13,11 +15,14 @@ export declare namespace Soil {
     };
     type LocalStackAwsService = never | 'acm' | 'apigateway' | 'cloudformation' | 'cloudwatch' | 'config' | 'dynamodb' | 'dynamodbstreams' | 'ec2' | 'es' | 'events' | 'firehose' | 'iam' | 'kinesis' | 'kms' | 'lambda' | 'logs' | 'opensearch' | 'redshift' | 'resource' | 'resourcegroupstaggingapi' | 'route53' | 'route53resolver' | 's3' | 's3control' | 'scheduler' | 'secretsmanager' | 'ses' | 'sns' | 'sqs' | 'ssm' | 'stepfunctions' | 'sts' | 'support' | 'swf' | 'transcribe';
     type BaseArgs = {
+        logger: Logger;
         registry: Registry<any>;
     };
     class Base {
+        protected logger: Logger;
         protected registry: Registry<any>;
         constructor(args: BaseArgs);
+        getRegion(): string;
         getTerraformPetals(ctx: Context): Promise<PetalProjResult>;
     }
     type LocalStackArgs = BaseArgs & {
@@ -34,19 +39,22 @@ export declare namespace Soil {
         private static localStackInternalPort;
         private aws;
         private localStackDocker;
-        private procArgs;
         constructor(args: LocalStackArgs);
+        getRegion(): "ca-central-1" | "us-east-1" | "us-east-2" | "us-west-1" | "us-west-2";
         private getAwsServices;
         private getDockerContainers;
-        run(args: {
-            logger: Logger;
-        }): Promise<{
+        run(): Promise<{
             aws: {
                 services: LocalStackAwsService[];
                 region: "ca-central-1" | "us-east-1" | "us-east-2" | "us-west-1" | "us-west-2";
             };
             netProc: NetProc;
-            url: string;
+            getApis: () => Promise<{
+                [x: string]: RestApi & {
+                    id: string;
+                    name: string;
+                };
+            }>;
         }>;
         end(args?: {
             containers?: Awaited<ReturnType<Soil.LocalStack['getDockerContainers']>>;
@@ -71,6 +79,7 @@ export declare namespace Soil {
     class AwsCloud extends Base {
         private aws;
         constructor(args: AwsCloudArgs);
+        getRegion(): "ca-central-1" | "us-east-1" | "us-east-2" | "us-west-1" | "us-west-2";
         getTerraformPetals(ctx: Context): Promise<{
             boot: () => Generator<PetalTerraform.Terraform | PetalTerraform.Provider | PetalTerraform.File, void, unknown>;
             main: (args: any) => Generator<PetalTerraform.Terraform | PetalTerraform.Provider | PetalTerraform.File, void, unknown>;