@gershy/lilac 0.0.12 → 0.0.13

package/cmp/mjs/main.js

@@ -7,12 +7,19 @@
 // Can region be dealt with any better??
 // Support test-mode (Flowers need to be able to do setup, share config, write to volumes, etc)
 import { PetalTerraform } from "./petal/terraform/terraform.js";
-import { regions as awsRegions } from "./util/aws.js";
 import { isCls, skip } from '@gershy/clearing';
 import procTerraform from "./util/procTerraform.js";
 import tryWithHealing from '@gershy/util-try-with-healing';
-const { File, Provider, Terraform } = PetalTerraform;
+import phrasing from '@gershy/util-phrasing';
 export class Flower {
+    // TODO: The downside of having this static is that different instances may use different
+    // services - e.g. api gateway instance may have "useEdge: true", in which case we'd like to
+    // include cloudfront and omit it otherwise... but having it on the instance is annoying since
+    // we want to enumerate all services *before* instantiating any Flowers... probably better this
+    // way? And heirarchical design can probably avoid most unecessary service inclusion...
+    // TODO: The naming of these services is coupled to LocalStack - consider using Lilac-scoped
+    // naming, and add a translation layer from Lilac->LocalStack in Soil.LocalStack?
+    static getAwsServices() { return []; }
     constructor() { }
     *getDependencies() {
         yield this;
@@ -23,10 +30,21 @@ export class Flower {
 }
 ;
 export class Registry {
+    // Note that maintaining a duality of classes for each Flower (one for testing, one for remote
+    // deploy) is essential to keep test functionality out of deployed code bundles. If a single
+    // class supported both test and prod functionality, these two pieces of functionality would
+    // always be bundled together.
     flowers;
     constructor(flowers) {
         this.flowers = {}[merge](flowers);
     }
+    getAwsServices() {
+        const services = new Set();
+        for (const [k, { real }] of this.flowers)
+            for (const awsService of real.getAwsServices())
+                services.add(awsService);
+        return services[toArr](v => v);
+    }
     add(flowers) {
         return new Registry({ ...this.flowers, ...flowers });
     }
@@ -38,23 +56,28 @@ export class Registry {
 export class Garden {
     // Note this class currently is coupled to terraform logic
     ctx;
-    registry;
-    define;
+    reg;
+    def;
     constructor(args) {
-        const { define, registry, ...ctx } = args;
-        this.ctx = ctx;
-        this.registry = registry;
-        this.define = define;
+        const { define, registry, context } = args;
+        this.ctx = context;
+        this.reg = registry;
+        this.def = define;
     }
-    async *getPetals(mode) {
+    async *getPetals() {
+        // TODO: We always use the "real" flowers from the registry - this is part of the shift to
+        // localStack; we always generate genuine terraform and apply it to the docker localStack.
+        // Eventually may want to support ultra-lightweight dockerless/localStackless js flower mocks;
+        // that would be the time to add "fake" flowers alongside each real flower, and start
+        // conditionally calling `this.registry.get('fake')`...
         const seenFlowers = new Set();
         const seenPetals = new Set();
-        for await (const topLevelFlower of this.define(this.ctx, this.registry.get(mode))) {
+        for await (const topLevelFlower of await this.def(this.ctx, this.reg.get('real'))) {
             for (const flower of topLevelFlower.getDependencies()) {
                 if (seenFlowers.has(flower))
                     continue;
                 seenFlowers.add(flower);
-                for await (const petal of flower.getPetals(this.ctx)) {
+                for await (const petal of await flower.getPetals(this.ctx)) {
                     if (seenPetals.has(petal))
                         continue;
                     yield petal;
@@ -62,8 +85,9 @@ export class Garden {
             }
         }
     }
-    async prepare( /* Note this only pertains to "real" mode */) {
-        return this.ctx.logger.scope('garden.prepare', {}, async (logger) => {
+    async genTerraform(deployTarget) {
+        const soilTfPetalsPrm = deployTarget.getTerraformPetals(this.ctx);
+        return this.ctx.logger.scope('garden.genTerraform', {}, async (logger) => {
             const setupTfProj = async (args) => args.logger.scope('tf', { proj: this.ctx.name, tf: args.term }, async (logger) => {
                 // Allows a terraform project to be defined in terms of a function which writes to main.tf,
                 // and adds any arbitrary additional files to the terraform project
@@ -82,11 +106,22 @@ export class Garden {
                 // Write new terraform
                 await logger.scope('files.generate', {}, async (logger) => {
                     const stream = await args.fact.kid(['main.tf']).getDataHeadStream();
-                    await args.setup(args.fact, stream);
+                    await args.setup(args.fact, stream, async (petal) => {
+                        // Include a utility function the caller can use to easily write petals
+                        const { tf, files = {} } = await petal.getResult()
+                            .then(tf => isCls(tf, String) ? { tf } : tf);
+                        if (tf)
+                            await stream.write(`${tf}\n`);
+                        await Promise.all(files[toArr]((data, kfp) => args.fact.kid(kfp.split('/')).setData(data)));
+                        return petal;
+                    });
                     await stream.end(); // TODO: @gershy/disk should allow `await headStream.end()`
                 });
                 return args.fact;
             });
+            // Pick names for the s3 and ddb terraform state persistence entities
+            const s3Name = `${this.ctx.pfx}-tf-state`;
+            const ddbName = `${this.ctx.pfx}-tf-state`;
             // We generate *two* terraform projects for every logical project - overall we want a
             // terraform project which saves its state in the cloud; in order to do this we need to first
             // provision the cloud storage engines to save the terraform state. The "boot" tf project
@@ -96,172 +131,106 @@ export class Garden {
                     term: 'boot',
                     logger,
                     fact: this.ctx.fact.kid(['boot']),
-                    setup: async (fact, mainWritable) => {
-                        await mainWritable.write(String[baseline](`
-              | terraform {
-              |   required_providers {
-              |     aws = {
-              |       source  = "hashicorp/aws"
-              |       version = "~> 5.0"
-              |     }
-              |   }
-              | }
-              | provider "aws" {
-              |   shared_credentials_files = [ "creds.ini" ]
-              |   profile                  = "default"
-              |   region                   = "ca-central-1"
-              | }
-              | resource "aws_s3_bucket" "tf_state" {
-              |   bucket = "${this.ctx.pfx}-tf-state"
-              | }
-              | resource "aws_s3_bucket_ownership_controls" "tf_state" {
-              |   bucket = aws_s3_bucket.tf_state.bucket
-              |   rule {
-              |     object_ownership = "ObjectWriter"
-              |   }
-              | }
-              | resource "aws_s3_bucket_acl" "tf_state" {
-              |   bucket = aws_s3_bucket.tf_state.bucket
-              |   acl = "private"
-              |   depends_on = [ aws_s3_bucket_ownership_controls.tf_state ]
-              | }
-              | resource "aws_dynamodb_table" "tf_state" {
-              |   name         = "${this.ctx.pfx}-tf-state"
-              |   billing_mode = "PAY_PER_REQUEST"
-              |   hash_key     = "LockID"
-              |   attribute {
-              |     name = "LockID"
-              |     type = "S"
-              |   }
-              | }
-            `));
-                        await fact.kid(['creds.ini']).setData(String[baseline](`
-              | [default]
-              | aws_region            = ${this.ctx.aws.region}
-              | aws_access_key_id     = ${this.ctx.aws.accessKey.id}
-              | aws_secret_access_key = ${this.ctx.aws.accessKey['!secret']}
-            `));
+                    setup: async (fact, mainWritable, writePetalTfAndFiles) => {
+                        // Include the soil's infrastructure
+                        const { boot } = await soilTfPetalsPrm;
+                        for await (const petal of await boot({ s3Name, ddbName }))
+                            await writePetalTfAndFiles(petal);
+                        // Create s3 tf state bucket
+                        const s3 = await writePetalTfAndFiles(new PetalTerraform.Resource('awsS3Bucket', 'tfState', {
+                            bucket: s3Name
+                        }));
+                        const s3Controls = await writePetalTfAndFiles(new PetalTerraform.Resource('awsS3BucketOwnershipControls', 'tfState', {
+                            bucket: s3.ref('bucket'),
+                            $rule: {
+                                objectOwnership: 'ObjectWriter'
+                            }
+                        }));
+                        await writePetalTfAndFiles(new PetalTerraform.Resource('awsS3BucketAcl', 'tfState', {
+                            bucket: s3.ref('bucket'),
+                            acl: 'private',
+                            dependsOn: [s3Controls.ref()]
+                        }));
+                        // Create ddb tf state locking table
+                        await writePetalTfAndFiles(new PetalTerraform.Resource('awsDynamodbTable', 'tfState', {
+                            name: ddbName,
+                            billingMode: phrasing('payPerRequest', 'camel', 'snake')[upper](),
+                            hashKey: 'LockID',
+                            $attribute: { name: 'LockID', type: 'S' }
+                        }));
                     }
                 }),
                 mainFact: setupTfProj({
                     term: 'main',
                     logger,
                     fact: this.ctx.fact.kid(['main']),
-                    setup: async (fact, mainWritable) => {
-                        const garden = this;
-                        const iteratePetals = async function* () {
-                            const tfAwsCredsFile = new File('creds.ini', String[baseline](`
-                | [default]
-                | aws_region            = ${garden.ctx.aws.region}
-                | aws_access_key_id     = ${garden.ctx.aws.accessKey.id}
-                | aws_secret_access_key = ${garden.ctx.aws.accessKey['!secret']}
-              `));
-                            yield tfAwsCredsFile;
-                            const terraform = new Terraform({
-                                $requiredProviders: {
-                                    aws: {
-                                        source: 'hashicorp/aws',
-                                        version: `~> 5.0` // Consider parameterizing??
-                                    }
-                                },
-                                '$backend.s3': {
-                                    region: garden.ctx.aws.region,
-                                    encrypt: true,
-                                    // Note references not allowed in terraform.backend!!
-                                    bucket: `${garden.ctx.pfx}-tf-state`,
-                                    key: `tf`,
-                                    dynamodbTable: `${garden.ctx.pfx}-tf-state`, // Dynamodb table is aws-account-wide
-                                    sharedCredentialsFiles: [tfAwsCredsFile.tfRef()],
-                                    profile: 'default', // References a section within the credentials file
-                                }
-                            });
-                            yield terraform;
-                            for (const { term } of awsRegions)
-                                yield new Provider('aws', {
-                                    sharedCredentialsFiles: [tfAwsCredsFile.tfRef()],
-                                    profile: 'default', // References a section within the credentials file
-                                    region: term,
-                                    // Omit the alias for the default provider!
-                                    ...(term !== garden.ctx.aws.region && { alias: term.split('-').join('_') })
-                                });
-                            yield* garden.getPetals('real');
-                        };
-                        const patioTfHclFact = garden.ctx.patioFact.kid(['main', '.terraform.lock.hcl']);
+                    setup: async (fact, mainWritable, writePetalTfAndFiles) => {
+                        // Include the soil's infrastructure
+                        const { main } = await soilTfPetalsPrm;
+                        for await (const petal of await main({ s3Name, ddbName }))
+                            await writePetalTfAndFiles(petal);
+                        for await (const petal of this.getPetals())
+                            await writePetalTfAndFiles(petal);
+                        // Propagate any terraform lock found in version control
+                        const patioTfHclFact = this.ctx.patioFact.kid(['main', '.terraform.lock.hcl']);
                         const tfHclData = await patioTfHclFact.getData('str');
                         if (tfHclData)
                             await fact.kid(['.terraform.lock.hcl']).setData(tfHclData);
-                        for await (const petal of iteratePetals()) {
-                            const result = await (async () => {
-                                const result = await petal.getResult();
-                                if (!isCls(result, Object))
-                                    return { tf: result, files: {} };
-                                return { files: {}, ...result };
-                            })();
-                            if (result.tf)
-                                await mainWritable.write(`${result.tf}\n`);
-                            await Promise.all(result.files[toArr]((data, kfp) => fact.kid(kfp.split('/')).setData(data)));
-                        }
                     }
-                })
+                }),
             });
         });
     }
     terraformInit(fact, args) {
-        return this.ctx.logger.scope('execTf.init', {}, async (logger) => {
+        return this.ctx.logger.scope('execTf.init', { fact: fact.fsp() }, async (logger) => {
             // Consider if we ever want to pass "-reconfigure" and "-migrate-state" options; these are
             // useful if we are moving backends (e.g. one aws account to another), and want to move our
             // full iac definition too
-            // TODO: Some terraform commands fail when offline - can this be covered up?
-            const result = await procTerraform(fact, `terraform init -input=false`);
+            // TODO: Some terraform commands fail when offline - can this be covered up? Possibly by
+            // checking terraform binaries into the repo? (Cross-platform nightmare though...)
+            const result = await procTerraform(fact, `terraform init -input=false`, {
+                onData: async (mode, data) => logger.log({ $$: 'notice', mode, data }) ?? null
+            });
             logger.log({ $$: 'result', logFp: result.logDb.toString(), msg: result.output });
             return result;
         });
     }
     terraformPlan(fact, args) {
-        return this.ctx.logger.scope('execTf.plan', {}, async (logger) => {
+        return this.ctx.logger.scope('execTf.plan', { fact: fact.fsp() }, async (logger) => {
             const result = await procTerraform(fact, `terraform plan -input=false`);
             logger.log({ $$: 'result', logFp: result.logDb.toString(), msg: result.output });
             return result;
         });
     }
     terraformApply(fact, args) {
-        return this.ctx.logger.scope('execTf.apply', {}, async (logger) => {
+        return this.ctx.logger.scope('execTf.apply', { fact: fact.fsp() }, async (logger) => {
             const result = await procTerraform(fact, `terraform apply -input=false -auto-approve`);
             logger.log({ $$: 'result', logFp: result.logDb.toString(), msg: result.output });
             return result;
         });
     }
-    async grow(mode) {
-        // - Avoid tf preventing deletions on populated dbs - e.g. s3 tf needs "force_destroy = true"
-        // - Note `terraform init` generates a ".terraform.lock.hcl" file - this should be checked into
-        //   consumer's source control! May need consumers to provide an optional `repoFact`, and if
-        //   present after `terraform init` we can copy the ".terraform.lock.hcl" file
-        // Note that test mode does not involve any iac - it all runs locally
-        if (mode === 'test')
-            throw Error('test mode not implemented yet');
-        const { bootFact, mainFact } = await this.prepare();
-        const logicalApply = (args) => {
-            return tryWithHealing({
-                fn: () => this.terraformApply(args.mainFact),
-                canHeal: err => (err.output ?? '')[has]('please run "terraform init"'),
+    async grow(deploy) {
+        if (deploy.type === 'test')
+            throw Error('not implemented')[mod]({ type: 'test' }); // TODO: Can be nice to have local service mocks!
+        const { bootFact, mainFact } = await this.genTerraform(deploy.soil);
+        // Init+apply both "boot" and "main", in optimistic fashion
+        const isHealableTerraformApply = err => /run[^a-zA-Z0-9]+terraform init/.test(err.output ?? '');
+        await tryWithHealing({
+            fn: () => this.terraformApply(mainFact),
+            canHeal: isHealableTerraformApply,
+            heal: () => tryWithHealing({
+                fn: async () => {
+                    await this.terraformInit(mainFact);
+                    await this.ctx.patioFact.kid(['main', '.terraform.lock.hcl']).setData(await mainFact.kid(['.terraform.lock.hcl']).getData('str'));
+                },
+                canHeal: err => true,
                 heal: () => tryWithHealing({
-                    fn: async () => {
-                        await this.terraformInit(args.mainFact);
-                        await this.ctx.patioFact.kid(['main', '.terraform.lock.hcl']).setData(await args.mainFact.kid(['.terraform.lock.hcl']).getData('str'));
-                    },
-                    canHeal: err => true,
-                    heal: () => tryWithHealing({
-                        fn: () => this.terraformApply(args.bootFact),
-                        canHeal: err => (err.output ?? '')[has]('please run "terraform init"'),
-                        heal: () => this.terraformInit(args.bootFact)
-                    })
+                    fn: () => this.terraformApply(bootFact),
+                    canHeal: isHealableTerraformApply,
+                    heal: () => this.terraformInit(bootFact)
                 })
-            });
-        };
-        // TODO: HEEERE do `logicalApply` instead!
-        await this.terraformInit(bootFact);
-        // const result = await logicalApply({ mainFact, bootFact });
-        //const result = await execTf.init(bootFact);
+            })
+        });
     }
 }
 ;

package/cmp/mjs/petal/terraform/terraform.d.ts

@@ -8,8 +8,8 @@ export declare namespace PetalTerraform {
         getProps(): {
             [key: string]: Json;
         };
-        tfRef(props?: string | string[]): string;
-        tfRefp(props?: string | string[]): `| ${string}`;
+        refStr(props?: string | string[]): string;
+        ref(props?: string | string[]): `| ${string}`;
         getResultHeader(): Promise<string>;
         getResult(): Promise<string | {
             tf: string;
@@ -64,7 +64,7 @@ export declare namespace PetalTerraform {
             [key: string]: Json;
         };
         getResultHeader(): Promise<string>;
-        tfRef(props?: string | string[]): string;
+        refStr(props?: string | string[]): string;
     }
     class File extends Base {
         private fp;
@@ -77,6 +77,6 @@ export declare namespace PetalTerraform {
                 [x: string]: string | Buffer<ArrayBufferLike>;
             };
         }>;
-        tfRef(props?: string | string[]): string;
+        refStr(props?: string | string[]): string;
     }
 }

package/cmp/mjs/petal/terraform/terraform.js

@@ -73,7 +73,7 @@ export var PetalTerraform;
         getType() { throw Error('not implemented'); }
         getHandle() { throw Error('not implemented'); }
         getProps() { throw Error('not implemented'); }
-        tfRef(props = []) {
+        refStr(props = []) {
             if (!isCls(props, Array))
                 props = [props];
             const base = `${ph(this.getType(), 'camel', 'snake')}.${ph(this.getHandle(), 'camel', 'snake')}`;
@@ -81,9 +81,9 @@ export var PetalTerraform;
                 ? `${base}.${props[map](v => ph(v, 'camel', 'snake')).join('.')}`
                 : base;
         }
-        tfRefp(props = []) {
+        ref(props = []) {
             // "plain ref" - uses "| " to avoid being  quoted within terraform
-            return `| ${this.tfRef(props)}`;
+            return `| ${this.refStr(props)}`;
         }
         async getResultHeader() { throw Error('not implemented'); }
         async getResult() {
@@ -160,8 +160,8 @@ export var PetalTerraform;
         async getResultHeader() {
             return `data "${ph(this.type, 'camel', 'snake')}" "${ph(this.handle, 'camel', 'snake')}"`;
         }
-        tfRef(props = []) {
-            return `data.${super.tfRef(props)}`;
+        refStr(props = []) {
+            return `data.${super.refStr(props)}`;
         }
     }
     PetalTerraform.Data = Data;
@@ -178,7 +178,7 @@ export var PetalTerraform;
         async getResult() {
             return { tf: '', files: { [this.fp]: this.content } };
         }
-        tfRef(props) {
+        refStr(props) {
             return this.fp; // `this.fp` should be quoted but not transformed to a tf handle
         }
     }

package/cmp/mjs/soil/soil.d.ts

@@ -0,0 +1,79 @@
+import { Context, PetalTerraform, Registry } from '../main.ts';
+import { RegionTerm } from '../util/aws.ts';
+import { NetProc } from '@gershy/util-http';
+import { SuperIterable } from '../util/superIterable.ts';
+import Logger from '@gershy/logger';
+export declare namespace Soil {
+    type PetalProjArgs = {
+        s3Name: string;
+        ddbName: string;
+    };
+    type PetalProjResult = {
+        [K in 'boot' | 'main']: (args: PetalProjArgs) => SuperIterable<PetalTerraform.Base>;
+    };
+    type LocalStackAwsService = never | 'acm' | 'apigateway' | 'cloudformation' | 'cloudwatch' | 'config' | 'dynamodb' | 'dynamodbstreams' | 'ec2' | 'es' | 'events' | 'firehose' | 'iam' | 'kinesis' | 'kms' | 'lambda' | 'logs' | 'opensearch' | 'redshift' | 'resource' | 'resourcegroupstaggingapi' | 'route53' | 'route53resolver' | 's3' | 's3control' | 'scheduler' | 'secretsmanager' | 'ses' | 'sns' | 'sqs' | 'ssm' | 'stepfunctions' | 'sts' | 'support' | 'swf' | 'transcribe';
+    type BaseArgs = {
+        registry: Registry<any>;
+    };
+    class Base {
+        protected registry: Registry<any>;
+        constructor(args: BaseArgs);
+        getTerraformPetals(ctx: Context): Promise<PetalProjResult>;
+    }
+    type LocalStackArgs = BaseArgs & {
+        aws: {
+            region: RegionTerm;
+        };
+        localStackDocker?: {
+            image?: `localstack/localstack${':' | ':latest' | '@'}${string}`;
+            containerName?: string;
+            port?: number;
+        };
+    };
+    class LocalStack extends Base {
+        private static localStackInternalPort;
+        private aws;
+        private localStackDocker;
+        private procArgs;
+        constructor(args: LocalStackArgs);
+        private getAwsServices;
+        private getDockerContainers;
+        run(args: {
+            logger: Logger;
+        }): Promise<{
+            aws: {
+                services: LocalStackAwsService[];
+                region: "ca-central-1" | "us-east-1" | "us-east-2" | "us-west-1" | "us-west-2";
+            };
+            netProc: NetProc;
+            url: string;
+        }>;
+        end(args?: {
+            containers?: Awaited<ReturnType<Soil.LocalStack['getDockerContainers']>>;
+        }): Promise<{
+            name: string;
+            state: "created" | "running" | "paused" | "restarting" | "removing" | "exited" | "dead";
+        }[]>;
+        getTerraformPetals(ctx: Context): Promise<{
+            boot: () => (PetalTerraform.Terraform | PetalTerraform.Provider)[];
+            main: (args: any) => Generator<PetalTerraform.Terraform | PetalTerraform.Provider, void, unknown>;
+        }>;
+    }
+    type AwsCloudArgs = BaseArgs & {
+        aws: {
+            region: RegionTerm;
+            accessKey: {
+                id: string;
+                '!secret': string;
+            };
+        };
+    };
+    class AwsCloud extends Base {
+        private aws;
+        constructor(args: AwsCloudArgs);
+        getTerraformPetals(ctx: Context): Promise<{
+            boot: () => Generator<PetalTerraform.Terraform | PetalTerraform.Provider | PetalTerraform.File, void, unknown>;
+            main: (args: any) => Generator<PetalTerraform.Terraform | PetalTerraform.Provider | PetalTerraform.File, void, unknown>;
+        }>;
+    }
+}