cloudmason2 1.0.0

package/commands/list_apps.js

@@ -0,0 +1,48 @@
+const { S3Client, ListObjectsV2Command } = require("@aws-sdk/client-s3");
+const OrgConfig = require('./helpers/org_config');
+
+// list-apps: every folder in the cdn bucket holding a manifest.json is an app
+exports.main = async function(){
+    const cfg = OrgConfig.read();
+    if (!cfg){ throw new Error('No org config found. Run set-org') }
+    const { bucket, prefix } = parseCdn(cfg.cdn);
+    const s3 = new S3Client({ region: cfg.region });
+
+    const base = prefix ? `${prefix}/` : '';
+    const apps = [];
+    let token;
+    do {
+        const r = await s3.send(new ListObjectsV2Command({
+            Bucket: bucket,
+            Prefix: base,
+            ContinuationToken: token
+        }));
+        (r.Contents || []).forEach(o=>{
+            const rel = o.Key.slice(base.length).split('/');
+            if (rel.length === 2 && rel[1] === 'manifest.json'){
+                const folder = rel[0];
+                apps.push(folder.startsWith('_') ? `${folder.slice(1)} (private)` : folder);
+            }
+        });
+        token = r.NextContinuationToken;
+    } while (token);
+
+    if (apps.length === 0){
+        console.log('No apps found in ' + bucket);
+        return true;
+    }
+    apps.sort().forEach(a=>{ console.log(a) });
+    console.log(`\n${apps.length} app${apps.length === 1 ? '' : 's'}`);
+    return true;
+}
+
+
+/////////////////////////////////////////
+////////////// FUNCS ////////////////////
+////////////////////////////////////////
+
+function parseCdn(cdn){
+    const p = cdn.trim().replace(/^s3:\/\//i, '').replace(/\/+$/, '');
+    const parts = p.split('/');
+    return { bucket: parts[0], prefix: parts.slice(1).join('/') };
+}

package/commands/new_app.js

@@ -0,0 +1,227 @@
+const fs = require('fs');
+const path = require('path');
+const { S3Client, PutObjectCommand, GetObjectCommand, ListObjectsV2Command } = require("@aws-sdk/client-s3");
+const { IAMClient, GetRoleCommand, CreateRoleCommand, PutRolePolicyCommand, GetOpenIDConnectProviderCommand, CreateOpenIDConnectProviderCommand } = require("@aws-sdk/client-iam");
+const { STSClient, GetCallerIdentityCommand } = require("@aws-sdk/client-sts");
+const OrgConfig = require('./helpers/org_config');
+
+const TEMPLATE_DIR = path.resolve(__dirname, '..', '.marketplace');
+
+// Base manifest for a new app
+const MANIFEST_TEMPLATE = {
+    name: "",
+    productId: "",
+    RecommendedInstanceType: "r8g.medium",
+    versions: []
+};
+
+// new-app: create the app's core assets in s3, the local template folder,
+// and (if -repo) the GitHub Actions CI role. Listing content is set by update-listing
+exports.main = async function(args){
+    const cfg = OrgConfig.read();
+    if (!cfg){ throw new Error('No org config found. Run set-org') }
+    const { bucket, prefix } = parseCdn(cfg.cdn);
+    const s3 = new S3Client({ region: cfg.region });
+
+    const appName = args.name.toLowerCase();
+    // -private: leading _ keeps the folder out of the CDN bucket's public policy
+    const folderName = (args.private !== undefined ? '_' : '') + appName;
+    const appPrefix = prefix ? `${prefix}/${folderName}` : folderName;
+
+    // --- I CORE ASSETS ---
+    const exists = await appFolderExists(s3, bucket, appPrefix);
+    if (exists){
+        console.log(`App folder exists: s3://${bucket}/${appPrefix}`);
+    } else {
+        await s3.send(new PutObjectCommand({ Bucket: bucket, Key: `${appPrefix}/` }));
+        console.log(`Created app folder s3://${bucket}/${appPrefix}`);
+        await createManifest(s3, bucket, appPrefix, args.name, args.noami !== undefined, args.private !== undefined);
+    }
+
+    // --- II LOCAL TEMPLATES ---
+    createLocalMarketplace();
+
+    // --- III REPO ROLE ---
+    if (args.repo){
+        if (!/^[\w.-]+\/[\w.-]+$/.test(args.repo)){ throw new Error('Invalid repo. Use org/repo format') }
+        const manifest = await getManifest(s3, bucket, appPrefix);
+        await ensureGithubRole(args.repo, bucket, appPrefix, appName, manifest.productId);
+    }
+    return true;
+}
+
+
+/////////////////////////////////////////
+////////////// FUNCS ////////////////////
+////////////////////////////////////////
+
+function parseCdn(cdn){
+    const p = cdn.trim().replace(/^s3:\/\//i, '').replace(/\/+$/, '');
+    const parts = p.split('/');
+    return { bucket: parts[0], prefix: parts.slice(1).join('/') };
+}
+
+async function appFolderExists(s3, bucket, appPrefix){
+    const r = await s3.send(new ListObjectsV2Command({
+        Bucket: bucket,
+        Prefix: `${appPrefix}/`,
+        MaxKeys: 1
+    }));
+    return r.KeyCount > 0;
+}
+
+async function createManifest(s3, bucket, appPrefix, appName, noami, isPrivate){
+    const manifest = JSON.parse(JSON.stringify(MANIFEST_TEMPLATE));
+    manifest.name = appName;
+    manifest.versions = [];
+    if (noami){ manifest.noami = true }
+    if (isPrivate){ manifest.private = true }
+    await s3.send(new PutObjectCommand({
+        Bucket: bucket,
+        Key: `${appPrefix}/manifest.json`,
+        Body: JSON.stringify(manifest, null, 4),
+        ContentType: 'application/json'
+    }));
+    console.log(`Created manifest s3://${bucket}/${appPrefix}/manifest.json`);
+}
+
+function createLocalMarketplace(){
+    const destDir = path.join(process.cwd(), '.marketplace');
+    if (!fs.existsSync(destDir)){ fs.mkdirSync(destDir, { recursive: true }) }
+
+    const srcDir = TEMPLATE_DIR;
+    if (destDir === srcDir){ return } // running from the cloudmason repo itself
+    fs.readdirSync(srcDir).forEach(f=>{
+        if (f.startsWith('.')){ return } // skip .DS_Store etc
+        const destFile = path.join(destDir, f);
+        if (fs.existsSync(destFile)){ return } // never overwrite edited assets
+        fs.copyFileSync(path.join(srcDir, f), destFile);
+    });
+    console.log(`Created ${destDir}`);
+}
+
+// Create a role GitHub Actions can assume via OIDC from the given repo.
+// Permissions cover update-listing, update-app, launch, and publish - not new-app
+async function ensureGithubRole(repo, bucket, appPrefix, appName, productId){
+    const roleName = `mason-gha-${appName}`;
+    const iam = new IAMClient({ region: 'us-east-1' });
+    const sts = new STSClient({ region: 'us-east-1' });
+    const accountId = (await sts.send(new GetCallerIdentityCommand({}))).Account;
+
+    const oidcArn = await ensureOidcProvider(iam, accountId);
+
+    try {
+        const existing = await iam.send(new GetRoleCommand({ RoleName: roleName }));
+        console.log(`Role ${roleName} already exists (${existing.Role.Arn})`);
+        return existing.Role.Arn;
+    } catch (e){
+        if (!/NoSuchEntity/.test(e)){ throw e }
+    }
+
+    // Trust: only GitHub Actions workflows from this repo
+    const trustPolicy = {
+        Version: "2012-10-17",
+        Statement: [
+            {
+                Effect: "Allow",
+                Principal: { Federated: oidcArn },
+                Action: "sts:AssumeRoleWithWebIdentity",
+                Condition: {
+                    StringEquals: { "token.actions.githubusercontent.com:aud": "sts.amazonaws.com" },
+                    StringLike: { "token.actions.githubusercontent.com:sub": `repo:${repo}:*` }
+                }
+            }
+        ]
+    };
+    const created = await iam.send(new CreateRoleCommand({
+        RoleName: roleName,
+        AssumeRolePolicyDocument: JSON.stringify(trustPolicy),
+        Description: `Mason CI role for ${appName} (github.com/${repo})`
+    }));
+
+    // Scoped to the app prefix, product id, and app stack names where the
+    // services support it. EC2 build permissions cannot be scoped
+    if (!productId){ console.log('WARNING: No productId in manifest. Marketplace access is scoped to all products. Re-create the role after update-listing -pid to scope it') }
+    const marketplaceEntity = productId
+        ? `arn:aws:aws-marketplace:us-east-1:${accountId}:AWSMarketplace/AmiProduct/${productId}`
+        : `arn:aws:aws-marketplace:us-east-1:${accountId}:AWSMarketplace/AmiProduct/*`;
+    const policy = {
+        Version: "2012-10-17",
+        Statement: [
+            {
+                Sid: "AppPrefixWrite",
+                Effect: "Allow",
+                Action: ["s3:GetObject", "s3:PutObject"],
+                Resource: `arn:aws:s3:::${bucket}/${appPrefix}/*`
+            },
+            {
+                Sid: "AppPrefixList",
+                Effect: "Allow",
+                Action: "s3:ListBucket",
+                Resource: `arn:aws:s3:::${bucket}`,
+                Condition: { StringLike: { "s3:prefix": `${appPrefix}/*` } }
+            },
+            {
+                Sid: "MarketplacePublish",
+                Effect: "Allow",
+                Action: ["aws-marketplace:StartChangeSet", "aws-marketplace:DescribeChangeSet", "aws-marketplace:DescribeEntity"],
+                Resource: [
+                    marketplaceEntity,
+                    `arn:aws:aws-marketplace:us-east-1:${accountId}:AWSMarketplace/ChangeSet/*`
+                ]
+            },
+            {
+                Sid: "AppStacks",
+                Effect: "Allow",
+                Action: ["cloudformation:CreateStack", "cloudformation:UpdateStack", "cloudformation:DescribeStacks"],
+                Resource: `arn:aws:cloudformation:*:${accountId}:stack/${appName}-*/*`
+            },
+            {
+                Sid: "ReadRoleParam",
+                Effect: "Allow",
+                Action: "ssm:GetParameter",
+                Resource: `arn:aws:ssm:us-east-1:${accountId}:parameter/mason-cdn-${bucket}`
+            },
+            {
+                Sid: "AmiBuildAndLaunch",
+                Effect: "Allow",
+                Action: ["ec2:*", "autoscaling:*", "elasticloadbalancing:*"],
+                Resource: "*"
+            }
+        ]
+    };
+    await iam.send(new PutRolePolicyCommand({
+        RoleName: roleName,
+        PolicyName: 'mason-app-access',
+        PolicyDocument: JSON.stringify(policy)
+    }));
+
+    console.log(`Created role ${roleName} (${created.Role.Arn})`);
+    console.log(`Use in GitHub Actions with: role-to-assume: ${created.Role.Arn}`);
+    return created.Role.Arn;
+}
+
+async function ensureOidcProvider(iam, accountId){
+    const arn = `arn:aws:iam::${accountId}:oidc-provider/token.actions.githubusercontent.com`;
+    try {
+        await iam.send(new GetOpenIDConnectProviderCommand({ OpenIDConnectProviderArn: arn }));
+        return arn;
+    } catch (e){
+        if (!/NoSuchEntity/.test(e)){ throw e }
+    }
+    await iam.send(new CreateOpenIDConnectProviderCommand({
+        Url: 'https://token.actions.githubusercontent.com',
+        ClientIDList: ['sts.amazonaws.com'],
+        ThumbprintList: ['6938fd4d98bab03faadb97b34396831e3780aea1']
+    }));
+    console.log('Created GitHub OIDC identity provider');
+    return arn;
+}
+
+async function getManifest(s3, bucket, appPrefix){
+    const r = await s3.send(new GetObjectCommand({
+        Bucket: bucket,
+        Key: `${appPrefix}/manifest.json`
+    }));
+    return JSON.parse(await r.Body.transformToString());
+}

package/commands/publish.js

@@ -0,0 +1,254 @@
+const { MarketplaceCatalogClient, StartChangeSetCommand, DescribeChangeSetCommand } = require("@aws-sdk/client-marketplace-catalog");
+const { S3Client, PutObjectCommand, GetObjectCommand } = require("@aws-sdk/client-s3");
+const { SSMClient, GetParameterCommand } = require("@aws-sdk/client-ssm");
+const OrgConfig = require('./helpers/org_config');
+const Apps = require('./helpers/apps');
+
+// The Marketplace Catalog API is only available in us-east-1
+const CATALOG_REGION = 'us-east-1';
+const DELIVERY_OPTION_TITLE = 'AMI with CloudFormation Template';
+
+// Manifest fields required by the AddDeliveryOptions change request
+const REQUIRED_MANIFEST = ['productId', 'ShortDescription', 'LongDescription', 'UsageInstructions', 'RecommendedInstanceType', 'ArchitectureDiagram'];
+
+exports.main = async function(args){
+    const cfg = OrgConfig.read();
+    if (!cfg){ throw new Error('No org config found. Run set-org') }
+    const { bucket, prefix } = parseCdn(cfg.cdn);
+    const s3 = new S3Client({ region: cfg.region });
+
+    const appName = args.app.toLowerCase();
+
+    // Get App + target version
+    const app = await Apps.resolveApp(s3, bucket, prefix, appName);
+    if (!app){ throw new Error('No app named ' + args.app + '. Run new-app') }
+    const { appPrefix, manifest } = app;
+    if (manifest.private){ throw new Error('Private apps cannot be published to marketplace') }
+    // -v accepts the major.minor or the full version name; the full name is used throughout
+    const version = manifest.versions.find(v=>{ return v.version === args.v || v.version.startsWith(args.v + '.') });
+    if (!version){ throw new Error(`No version ${args.v} of ${args.app}. Run update-app`) }
+    const fullVersion = version.version;
+    const vFolder = fullVersion.split('.').slice(0, 2).join('.'); // storage folder is major.minor
+
+    // --- I CHECK REQUIRED FIELDS ---
+    const role = await getMarketplaceRole(bucket);
+    if (!role){ throw new Error('No marketplace access role found for this CDN. Run create-cdn first') }
+    const missing = REQUIRED_MANIFEST.filter(k=>{ return !manifest[k] });
+    if (!version.ami_id){ missing.push('versions[].ami_id') }
+    if (!version.stack_url){ missing.push('versions[].stack_url') }
+    if (missing.length){
+        throw new Error('Missing required manifest fields: ' + missing.join(', ') + '. Run update-listing to set them');
+    }
+    const arch = /[0-9]g/.test(manifest.RecommendedInstanceType.split('.')[0]) ? 'arm' : 'x86_64';
+
+    console.log(`Publishing ${args.app} v${version.version} to product ${manifest.productId}`);
+
+    // --- II CREATE UPGRADE TEMPLATE ---
+    // Copy of the stack with the AmiId parameter stripped and refs hardcoded to the
+    // region-resolving marketplace SSM alias, for buyers upgrading an existing stack
+    const stackKey = `${appPrefix}/versions/${vFolder}/stack.yaml`;
+    const stackText = await getFileText(s3, bucket, stackKey);
+    if (stackText === null){ throw new Error('No stack found for v' + fullVersion) }
+
+    const amiAlias = `resolve:ssm:/aws/service/marketplace/${manifest.productId}/${fullVersion}`;
+    const upgradeText = buildUpgradeTemplate(stackText, amiAlias);
+    const upgradeKey = `${appPrefix}/versions/${vFolder}/upgrade.yaml`;
+    await s3.send(new PutObjectCommand({ Bucket: bucket, Key: upgradeKey, Body: upgradeText }));
+    console.log(`Uploaded upgrade template s3://${bucket}/${upgradeKey}`);
+
+    // --- III START CHANGE SET ---
+    const client = new MarketplaceCatalogClient({ region: CATALOG_REGION });
+    const changeSet = {
+        Catalog: "AWSMarketplace",
+        Intent: "APPLY",
+        ChangeSet: [
+            {
+                ChangeType: "AddDeliveryOptions",
+                Entity: {
+                    Type: "AmiProduct@1.0",
+                    Identifier: manifest.productId
+                },
+                Details: JSON.stringify({
+                    Version: {
+                        VersionTitle: fullVersion,
+                        ReleaseNotes: args.desc
+                    },
+                    DeliveryOptions: [
+                        {
+                            DeliveryOptionTitle: DELIVERY_OPTION_TITLE,
+                            Details: {
+                                "DeploymentTemplateDeliveryOptionDetails": {
+                                    "ShortDescription": manifest.ShortDescription,
+                                    "LongDescription": manifest.LongDescription,
+                                    "UsageInstructions": manifest.UsageInstructions,
+                                    "RecommendedInstanceType": manifest.RecommendedInstanceType,
+                                    "ArchitectureDiagram": manifest.ArchitectureDiagram,
+                                    "Template": version.stack_url,
+                                    "TemplateSources": [
+                                        {
+                                            "ParameterName": "AmiId",
+                                            "AmiSource": {
+                                                "AmiId": version.ami_id,
+                                                "AccessRoleArn": role,
+                                                "UserName": "ec2-user",
+                                                "OperatingSystemName": "AMAZONLINUX",
+                                                "OperatingSystemVersion": arch === 'arm'
+                                                    ? "Amazon Linux 2023 arm64 HVM"
+                                                    : "Amazon Linux 2023 x86_64 HVM"
+                                            }
+                                        }
+                                    ]
+                                }
+                            }
+                        }
+                    ]
+                })
+            }
+        ]
+    };
+    const startResponse = await client.send(new StartChangeSetCommand(changeSet));
+    const changeSetId = startResponse.ChangeSetId;
+    console.log('Change set started:', changeSetId);
+
+    // --- IV UPDATE MANIFEST ---
+    if (version.published !== true){
+        version.published = true;
+        version.publish_date = new Date().toISOString();
+        await putManifest(s3, bucket, appPrefix, manifest);
+        console.log(`Marked v${version.version} published in manifest`);
+    } else {
+        console.log(`v${version.version} already marked published (${version.publish_date})`);
+    }
+
+    // --- V AWAIT ---
+    if (args.await !== undefined){
+        const status = await awaitChangeSet(client, changeSetId);
+        if (status === 'FAILED'){
+            version.published = false;
+            await putManifest(s3, bucket, appPrefix, manifest);
+            throw new Error('Publish change set failed');
+        }
+        if (status === 'TIMEOUT'){
+            console.log('Change set still in progress after 90 minutes. Check the marketplace portal for final status');
+        }
+    } else {
+        console.log('Publish requested. Run with -await to wait for marketplace processing');
+    }
+    return true;
+}
+
+
+///////////////////////////////////////////////
+///////////////////////////////////////////////
+///////////////////////////////////////////////
+
+function parseCdn(cdn){
+    const p = cdn.trim().replace(/^s3:\/\//i, '').replace(/\/+$/, '');
+    const parts = p.split('/');
+    return { bucket: parts[0], prefix: parts.slice(1).join('/') };
+}
+
+// Poll the change set every 30s for up to 90 minutes
+async function awaitChangeSet(client, changeSetId){
+    for (let i=0; i<180; i++){
+        await new Promise(res=>{ setTimeout(res, 30000) });
+        const r = await client.send(new DescribeChangeSetCommand({
+            Catalog: "AWSMarketplace",
+            ChangeSetId: changeSetId
+        }));
+        console.log(`Change set status @${Math.round((i+1)*0.5)}m: ${r.Status}`);
+        if (r.Status === 'SUCCEEDED'){ return 'SUCCEEDED' }
+        if (r.Status === 'FAILED' || r.Status === 'CANCELLED'){
+            (r.ChangeSet || []).forEach(c=>{
+                (c.ErrorDetailList || []).forEach(e=>{ console.log(`\t${e.ErrorCode}: ${e.ErrorMessage}`) });
+            });
+            return 'FAILED';
+        }
+    }
+    return 'TIMEOUT';
+}
+
+// Strip the AmiId parameter and hardcode all references to the marketplace SSM alias
+function buildUpgradeTemplate(stackText, amiAlias){
+    if (stackText.trim().startsWith('{')){
+        const tpl = JSON.parse(stackText);
+        if (!tpl.Parameters || !tpl.Parameters.AmiId){ throw new Error('Stack has no AmiId parameter') }
+        delete tpl.Parameters.AmiId;
+        if (Object.keys(tpl.Parameters).length === 0){ delete tpl.Parameters }
+        return JSON.stringify(replaceAmiRefs(tpl, amiAlias), null, 4);
+    }
+
+    let lines = stackText.split(/\r?\n/);
+    const pStart = lines.findIndex(l=>{ return /^Parameters:\s*(#.*)?$/.test(l) });
+    let amiLine = -1;
+    let amiIndent = 0;
+    if (pStart > -1){
+        for (let i=pStart+1; i<lines.length; i++){
+            if (/^\S/.test(lines[i])){ break }
+            const m = lines[i].match(/^(\s+)AmiId:\s*(#.*)?$/);
+            if (m){ amiLine = i; amiIndent = m[1].length; break }
+        }
+    }
+    if (amiLine === -1){ throw new Error('Stack has no AmiId parameter') }
+
+    // Remove the AmiId parameter block
+    let blockEnd = lines.length;
+    for (let i=amiLine+1; i<lines.length; i++){
+        if (lines[i].trim() === ''){ continue }
+        if (lines[i].match(/^\s*/)[0].length <= amiIndent){ blockEnd = i; break }
+    }
+    lines.splice(amiLine, blockEnd - amiLine);
+
+    // Drop the Parameters section entirely if AmiId was its only entry
+    const nextLine = lines[pStart+1];
+    if (nextLine === undefined || /^\S/.test(nextLine)){ lines.splice(pStart, 1) }
+
+    // Hardcode references to the region-resolving alias
+    return lines.join('\n')
+        .replace(/!Ref\s+AmiId\b/g, `'${amiAlias}'`)
+        .replace(/Ref:\s*AmiId\b/g, `'${amiAlias}'`);
+}
+
+function replaceAmiRefs(obj, amiAlias){
+    if (Array.isArray(obj)){ return obj.map(o=>{ return replaceAmiRefs(o, amiAlias) }) }
+    if (obj && typeof obj === 'object'){
+        const keys = Object.keys(obj);
+        if (keys.length === 1 && keys[0] === 'Ref' && obj.Ref === 'AmiId'){ return amiAlias }
+        const out = {};
+        keys.forEach(k=>{ out[k] = replaceAmiRefs(obj[k], amiAlias) });
+        return out;
+    }
+    return obj;
+}
+
+// The role param is named per CDN bucket so one account can hold multiple cdns/roles.
+// create-cdn stores it in us-east-1
+async function getMarketplaceRole(bucket){
+    const ssm = new SSMClient({ region: 'us-east-1' });
+    try {
+        const r = await ssm.send(new GetParameterCommand({ Name: `mason-cdn-${bucket}` }));
+        return r.Parameter.Value;
+    } catch (e){
+        if (/ParameterNotFound/.test(e)){ return null }
+        throw e;
+    }
+}
+
+async function getFileText(s3, bucket, key){
+    try {
+        const r = await s3.send(new GetObjectCommand({ Bucket: bucket, Key: key }));
+        return await r.Body.transformToString();
+    } catch (e){
+        if (/NoSuchKey|NotFound/.test(e)){ return null }
+        throw e;
+    }
+}
+
+async function putManifest(s3, bucket, appPrefix, manifest){
+    await s3.send(new PutObjectCommand({
+        Bucket: bucket,
+        Key: `${appPrefix}/manifest.json`,
+        Body: JSON.stringify(manifest, null, 4),
+        ContentType: 'application/json'
+    }));
+}