@imgly/plugin-background-removal-web 0.1.0 → 0.2.0-rc.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE.md +652 -1
- package/README.md +105 -20
- package/dist/index.mjs +1 -491
- package/dist/index.mjs.map +4 -4
- package/dist/plugin.d.ts +5 -2
- package/dist/processBackgroundRemoval.d.ts +27 -1
- package/dist/types.d.ts +4 -38
- package/package.json +8 -6
- package/dist/constants.d.ts +0 -4
- package/dist/enableFeatures.d.ts +0 -6
- package/dist/registerComponents.d.ts +0 -6
- package/dist/utils.d.ts +0 -46
package/README.md
CHANGED
|
@@ -2,14 +2,15 @@
|
|
|
2
2
|
|
|
3
3
|
|
|
4
4
|
|
|
5
|
-
This plugin introduces background removal for the CE.SDK editor, leveraging the power of the [background-removal-js library](https://github.com/imgly/background-removal-js). It integrates seamlessly with CE.SDK,
|
|
5
|
+
This plugin introduces background removal for the CE.SDK editor, leveraging the power of the [background-removal-js library](https://github.com/imgly/background-removal-js). It integrates seamlessly with CE.SDK, provides users with an efficient tool to remove backgrounds from images directly in the browser with ease and no additional costs or privacy concerns.
|
|
6
6
|
|
|
7
7
|
## Installation
|
|
8
8
|
|
|
9
9
|
You can install the plugin via npm or yarn. Use the following commands to install the package:
|
|
10
10
|
|
|
11
11
|
```
|
|
12
|
-
yarn
|
|
12
|
+
yarn add @imgly/plugin-background-removal-web
|
|
13
|
+
npm install @imgly/plugin-background-removal-web
|
|
13
14
|
```
|
|
14
15
|
|
|
15
16
|
## Usage
|
|
@@ -19,44 +20,48 @@ canvas menu entry for every block with an image fill.
|
|
|
19
20
|
|
|
20
21
|
```typescript
|
|
21
22
|
import CreativeEditorSDK from '@cesdk/cesdk-js';
|
|
22
|
-
import BackgroundRemovalPlugin from '@imgly/plugin-background-removal';
|
|
23
|
+
import BackgroundRemovalPlugin from '@imgly/plugin-background-removal-web';
|
|
23
24
|
|
|
24
25
|
const config = {
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
26
|
+
license: '<your-license-here>',
|
|
27
|
+
callbacks: {
|
|
28
|
+
// Please note that the background removal plugin depends on an correctly
|
|
29
|
+
// configured upload. 'local' will work for local testing, but in
|
|
30
|
+
// production you will need something stable. Please take a look at:
|
|
31
|
+
// https://img.ly/docs/cesdk/ui/guides/upload-images/
|
|
32
|
+
onUpload: 'local'
|
|
33
|
+
}
|
|
33
34
|
};
|
|
34
35
|
|
|
35
36
|
const cesdk = await CreativeEditorSDK.create(container, config);
|
|
36
37
|
await cesdk.addDefaultAssetSources(),
|
|
37
|
-
|
|
38
|
-
|
|
38
|
+
await cesdk.addDemoAssetSources({ sceneMode: 'Design' }),
|
|
39
|
+
await cesdk.unstable_addPlugin(BackgroundRemovalPlugin());
|
|
39
40
|
|
|
40
41
|
await cesdk.createDesignScene();
|
|
41
42
|
```
|
|
42
43
|
|
|
43
44
|
## Configuration
|
|
44
45
|
|
|
45
|
-
|
|
46
|
-
|
|
46
|
+
By default, this plugin uses the `@imgly/background-removal-js` library to remove
|
|
47
|
+
a background from the image fill. All configuration options from this underlying
|
|
48
|
+
library can be used in this plugin.
|
|
47
49
|
|
|
48
50
|
[See the documentation](https://github.com/imgly/background-removal-js/tree/main/packages/web#advanced-configuration) for further information.
|
|
49
51
|
|
|
50
52
|
```typescript
|
|
51
|
-
import BackgroundRemovalPlugin from '@imgly/plugin-background-removal';
|
|
53
|
+
import BackgroundRemovalPlugin from '@imgly/plugin-background-removal-web';
|
|
52
54
|
|
|
53
55
|
[...]
|
|
54
56
|
|
|
55
57
|
await cesdk.unstable_addPlugin(BackgroundRemovalPlugin({
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
58
|
+
provider: {
|
|
59
|
+
type: '@imgly/background-removal',
|
|
60
|
+
configuration: {
|
|
61
|
+
publicPath: '...',
|
|
62
|
+
// All other configuration options that are passed to the bg removal
|
|
63
|
+
// library. See https://github.com/imgly/background-removal-js/tree/main/packages/web#advanced-configuration
|
|
64
|
+
}
|
|
60
65
|
}
|
|
61
66
|
}))
|
|
62
67
|
|
|
@@ -70,3 +75,83 @@ For optimal performance using the correct CORS headers is important. See the lib
|
|
|
70
75
|
'Cross-Origin-Opener-Policy': 'same-origin',
|
|
71
76
|
'Cross-Origin-Embedder-Policy': 'require-corp'
|
|
72
77
|
```
|
|
78
|
+
|
|
79
|
+
## Custom Background Removal Provider
|
|
80
|
+
|
|
81
|
+
It is possible to declare a different provider for the background removal process.
|
|
82
|
+
|
|
83
|
+
```typescript
|
|
84
|
+
BackgroundRemovalPlugin({
|
|
85
|
+
provider: {
|
|
86
|
+
type: 'custom',
|
|
87
|
+
// If the image has only one image file URI defined, this method will
|
|
88
|
+
// be called. It must return a single new image file URI with the
|
|
89
|
+
// removed background.
|
|
90
|
+
processImageFileURI: async (imageFileURI: string) => {
|
|
91
|
+
const blob = await removeBackground(imageFileURI);
|
|
92
|
+
const upload = await uploadBlob(blob);
|
|
93
|
+
return upload;
|
|
94
|
+
},
|
|
95
|
+
// Some images have a source set defined which provides multiple images
|
|
96
|
+
// in different sizes.
|
|
97
|
+
processSourceSet: async (
|
|
98
|
+
// Source set for the current block sorted by the resolution.
|
|
99
|
+
// URI with the highest URI is first
|
|
100
|
+
sourceSet: {
|
|
101
|
+
uri: string;
|
|
102
|
+
width: number;
|
|
103
|
+
height: number;
|
|
104
|
+
}[],
|
|
105
|
+
) => {
|
|
106
|
+
// You should call the remove background method on every URI in the
|
|
107
|
+
// source set. Depending on your service or your algorithm, you
|
|
108
|
+
// have the following options:
|
|
109
|
+
// - Return a source set with a single image (will contradict the use-case of source sets and degrades the user experience)
|
|
110
|
+
// - Create a segmented mask and apply it to every image (not always available)
|
|
111
|
+
// - Create a new source set by resizing the resulting blob.
|
|
112
|
+
|
|
113
|
+
// In this example we will do the last case.
|
|
114
|
+
// First image has the highest resolution and might be the best
|
|
115
|
+
// candidate to remove the background.
|
|
116
|
+
const highestResolution = sourceSet[0];
|
|
117
|
+
const highestResolutionBlob = await removeBackground(highestResolution.uri);
|
|
118
|
+
const highestResolutionURI = await uploadBlob(highestResolutionBlob);
|
|
119
|
+
|
|
120
|
+
const remainingSources = await Promise.all(
|
|
121
|
+
sourceSet.slice(1).map((source) => {
|
|
122
|
+
// ...
|
|
123
|
+
const upload = uploadBlob(/* ... */)
|
|
124
|
+
return { ...source, uri: upload };
|
|
125
|
+
})
|
|
126
|
+
);
|
|
127
|
+
|
|
128
|
+
return [{ ...highestResolution, uri: highestResolutionURI }, remainingSources];
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
});
|
|
132
|
+
```
|
|
133
|
+
|
|
134
|
+
Depending on your use case or service you might end up with a blob that you want to upload by the
|
|
135
|
+
configured upload handler of the editor. This might look like the following function:
|
|
136
|
+
|
|
137
|
+
```typescript
|
|
138
|
+
async function uploadBlob(
|
|
139
|
+
blob: Blob,
|
|
140
|
+
initialUri: string,
|
|
141
|
+
cesdk: CreativeEditorSDK
|
|
142
|
+
) {
|
|
143
|
+
const pathname = new URL(initialUri).pathname;
|
|
144
|
+
const parts = pathname.split('/');
|
|
145
|
+
const filename = parts[parts.length - 1];
|
|
146
|
+
|
|
147
|
+
const uploadedAssets = await cesdk.unstable_upload(
|
|
148
|
+
new File([blob], filename, { type: blob.type })
|
|
149
|
+
);
|
|
150
|
+
|
|
151
|
+
const url = uploadedAssets.meta?.uri;
|
|
152
|
+
if (url == null) {
|
|
153
|
+
throw new Error('Could not upload processed fill');
|
|
154
|
+
}
|
|
155
|
+
return url;
|
|
156
|
+
}
|
|
157
|
+
```
|
package/dist/index.mjs
CHANGED
|
@@ -1,492 +1,2 @@
|
|
|
1
|
-
//
|
|
2
|
-
import isEqual from "lodash/isEqual";
|
|
3
|
-
|
|
4
|
-
// src/constants.ts
|
|
5
|
-
var BG_REMOVAL_ID = "@imgly/plugin-background-removal-web";
|
|
6
|
-
var CANVAS_MENU_COMPONENT_ID = `${BG_REMOVAL_ID}.canvasMenu`;
|
|
7
|
-
var CANVAS_MENU_COMPONENT_BUTTON_ID = `${CANVAS_MENU_COMPONENT_ID}.button`;
|
|
8
|
-
var FEATURE_ID = `${BG_REMOVAL_ID}.feature`;
|
|
9
|
-
|
|
10
|
-
// src/utils.ts
|
|
11
|
-
function setBGRemovalMetadata(cesdk, id, metadata) {
|
|
12
|
-
cesdk.engine.block.setMetadata(id, BG_REMOVAL_ID, JSON.stringify(metadata));
|
|
13
|
-
}
|
|
14
|
-
function getBGRemovalMetadata(cesdk, id) {
|
|
15
|
-
if (cesdk.engine.block.hasMetadata(id, BG_REMOVAL_ID)) {
|
|
16
|
-
return JSON.parse(cesdk.engine.block.getMetadata(id, BG_REMOVAL_ID));
|
|
17
|
-
} else {
|
|
18
|
-
return {
|
|
19
|
-
status: "IDLE"
|
|
20
|
-
};
|
|
21
|
-
}
|
|
22
|
-
}
|
|
23
|
-
function clearBGRemovalMetadata(cesdk, id) {
|
|
24
|
-
if (cesdk.engine.block.hasMetadata(id, BG_REMOVAL_ID)) {
|
|
25
|
-
cesdk.engine.block.removeMetadata(id, BG_REMOVAL_ID);
|
|
26
|
-
}
|
|
27
|
-
}
|
|
28
|
-
function isDuplicate(cesdk, blockId, metadata) {
|
|
29
|
-
if (!cesdk.engine.block.isValid(blockId))
|
|
30
|
-
return false;
|
|
31
|
-
if (metadata.status === "IDLE" || metadata.status === "PENDING" || metadata.status === "ERROR")
|
|
32
|
-
return false;
|
|
33
|
-
if (!cesdk.engine.block.hasFill(blockId))
|
|
34
|
-
return false;
|
|
35
|
-
const fillId = cesdk.engine.block.getFill(blockId);
|
|
36
|
-
if (metadata.blockId === blockId || metadata.fillId === fillId)
|
|
37
|
-
return false;
|
|
38
|
-
return true;
|
|
39
|
-
}
|
|
40
|
-
function fixDuplicateMetadata(cesdk, blockId) {
|
|
41
|
-
const fillId = cesdk.engine.block.getFill(blockId);
|
|
42
|
-
const metadata = getBGRemovalMetadata(cesdk, blockId);
|
|
43
|
-
if (metadata.status === "IDLE" || metadata.status === "PENDING" || metadata.status === "ERROR")
|
|
44
|
-
return;
|
|
45
|
-
setBGRemovalMetadata(cesdk, blockId, {
|
|
46
|
-
...metadata,
|
|
47
|
-
blockId,
|
|
48
|
-
fillId
|
|
49
|
-
});
|
|
50
|
-
}
|
|
51
|
-
function isMetadataConsistent(cesdk, blockId) {
|
|
52
|
-
if (!cesdk.engine.block.isValid(blockId))
|
|
53
|
-
return false;
|
|
54
|
-
const metadata = getBGRemovalMetadata(cesdk, blockId);
|
|
55
|
-
if (metadata.status === "IDLE" || metadata.status === "PENDING")
|
|
56
|
-
return true;
|
|
57
|
-
if (!cesdk.engine.block.hasFill(blockId))
|
|
58
|
-
return false;
|
|
59
|
-
const fillId = cesdk.engine.block.getFill(blockId);
|
|
60
|
-
if (fillId == null)
|
|
61
|
-
return false;
|
|
62
|
-
if (blockId !== metadata.blockId || fillId !== metadata.fillId)
|
|
63
|
-
return false;
|
|
64
|
-
const sourceSet = cesdk.engine.block.getSourceSet(
|
|
65
|
-
fillId,
|
|
66
|
-
"fill/image/sourceSet"
|
|
67
|
-
);
|
|
68
|
-
const imageFileURI = cesdk.engine.block.getString(
|
|
69
|
-
fillId,
|
|
70
|
-
"fill/image/imageFileURI"
|
|
71
|
-
);
|
|
72
|
-
if (sourceSet.length === 0 && !imageFileURI && metadata.status === "PROCESSING") {
|
|
73
|
-
return true;
|
|
74
|
-
}
|
|
75
|
-
if (sourceSet?.length > 0) {
|
|
76
|
-
const initialSourceSet = metadata.initialSourceSet;
|
|
77
|
-
if (metadata.status === "PROCESSED_WITH_BG" || metadata.status === "PROCESSED_WITHOUT_BG") {
|
|
78
|
-
const removedBackground = metadata.removedBackground;
|
|
79
|
-
if (!isEqual(sourceSet, removedBackground) && !isEqual(sourceSet, initialSourceSet)) {
|
|
80
|
-
return false;
|
|
81
|
-
}
|
|
82
|
-
} else {
|
|
83
|
-
if (!isEqual(sourceSet, initialSourceSet)) {
|
|
84
|
-
return false;
|
|
85
|
-
}
|
|
86
|
-
}
|
|
87
|
-
} else {
|
|
88
|
-
if (metadata.status === "PROCESSED_WITH_BG" || metadata.status === "PROCESSED_WITHOUT_BG") {
|
|
89
|
-
if (imageFileURI !== metadata.initialImageFileURI && imageFileURI !== metadata.removedBackground) {
|
|
90
|
-
return false;
|
|
91
|
-
}
|
|
92
|
-
} else {
|
|
93
|
-
if (imageFileURI !== metadata.initialImageFileURI) {
|
|
94
|
-
return false;
|
|
95
|
-
}
|
|
96
|
-
}
|
|
97
|
-
}
|
|
98
|
-
return true;
|
|
99
|
-
}
|
|
100
|
-
function toggleBackgroundRemovalData(cesdk, blockId) {
|
|
101
|
-
const blockApi = cesdk.engine.block;
|
|
102
|
-
if (!blockApi.hasFill(blockId))
|
|
103
|
-
return;
|
|
104
|
-
const fillId = blockApi.getFill(blockId);
|
|
105
|
-
const metadata = getBGRemovalMetadata(cesdk, blockId);
|
|
106
|
-
if (metadata.status === "PROCESSED_WITH_BG") {
|
|
107
|
-
setBGRemovalMetadata(cesdk, blockId, {
|
|
108
|
-
...metadata,
|
|
109
|
-
status: "PROCESSED_WITHOUT_BG"
|
|
110
|
-
});
|
|
111
|
-
if (typeof metadata.removedBackground === "string") {
|
|
112
|
-
blockApi.setString(
|
|
113
|
-
fillId,
|
|
114
|
-
"fill/image/imageFileURI",
|
|
115
|
-
metadata.removedBackground
|
|
116
|
-
);
|
|
117
|
-
} else {
|
|
118
|
-
blockApi.setSourceSet(
|
|
119
|
-
fillId,
|
|
120
|
-
"fill/image/sourceSet",
|
|
121
|
-
metadata.removedBackground
|
|
122
|
-
);
|
|
123
|
-
}
|
|
124
|
-
cesdk.engine.editor.addUndoStep();
|
|
125
|
-
} else if (metadata.status === "PROCESSED_WITHOUT_BG") {
|
|
126
|
-
setBGRemovalMetadata(cesdk, blockId, {
|
|
127
|
-
...metadata,
|
|
128
|
-
status: "PROCESSED_WITH_BG"
|
|
129
|
-
});
|
|
130
|
-
blockApi.setString(
|
|
131
|
-
fillId,
|
|
132
|
-
"fill/image/imageFileURI",
|
|
133
|
-
metadata.initialImageFileURI
|
|
134
|
-
);
|
|
135
|
-
blockApi.setSourceSet(
|
|
136
|
-
fillId,
|
|
137
|
-
"fill/image/sourceSet",
|
|
138
|
-
metadata.initialSourceSet
|
|
139
|
-
);
|
|
140
|
-
cesdk.engine.editor.addUndoStep();
|
|
141
|
-
}
|
|
142
|
-
}
|
|
143
|
-
function recoverInitialImageData(cesdk, blockId) {
|
|
144
|
-
const blockApi = cesdk.engine.block;
|
|
145
|
-
if (!blockApi.hasFill(blockId))
|
|
146
|
-
return;
|
|
147
|
-
const metadata = getBGRemovalMetadata(cesdk, blockId);
|
|
148
|
-
if (metadata.status === "PENDING" || metadata.status === "IDLE") {
|
|
149
|
-
return;
|
|
150
|
-
}
|
|
151
|
-
const initialSourceSet = metadata.initialSourceSet;
|
|
152
|
-
const initialImageFileURI = metadata.initialImageFileURI;
|
|
153
|
-
const fillId = getValidFill(cesdk, blockId, metadata);
|
|
154
|
-
if (fillId == null)
|
|
155
|
-
return;
|
|
156
|
-
if (initialImageFileURI) {
|
|
157
|
-
cesdk.engine.block.setString(
|
|
158
|
-
fillId,
|
|
159
|
-
"fill/image/imageFileURI",
|
|
160
|
-
initialImageFileURI
|
|
161
|
-
);
|
|
162
|
-
}
|
|
163
|
-
if (initialSourceSet.length > 0) {
|
|
164
|
-
cesdk.engine.block.setSourceSet(
|
|
165
|
-
fillId,
|
|
166
|
-
"fill/image/sourceSet",
|
|
167
|
-
initialSourceSet
|
|
168
|
-
);
|
|
169
|
-
}
|
|
170
|
-
}
|
|
171
|
-
function getValidFill(cesdk, blockId, metadata) {
|
|
172
|
-
if (!cesdk.engine.block.isValid(blockId) || !cesdk.engine.block.hasFill(blockId) || blockId !== metadata.blockId) {
|
|
173
|
-
return void 0;
|
|
174
|
-
}
|
|
175
|
-
const fillId = cesdk.engine.block.getFill(blockId);
|
|
176
|
-
if (fillId !== metadata.fillId) {
|
|
177
|
-
return void 0;
|
|
178
|
-
}
|
|
179
|
-
return fillId;
|
|
180
|
-
}
|
|
181
|
-
|
|
182
|
-
// src/registerComponents.ts
|
|
183
|
-
function registerComponents(cesdk) {
|
|
184
|
-
cesdk.ui.unstable_setCanvasMenuOrder([
|
|
185
|
-
CANVAS_MENU_COMPONENT_ID,
|
|
186
|
-
...cesdk.ui.unstable_getCanvasMenuOrder()
|
|
187
|
-
]);
|
|
188
|
-
cesdk.ui.unstable_registerComponent(
|
|
189
|
-
CANVAS_MENU_COMPONENT_ID,
|
|
190
|
-
({ builder: { Button }, engine }) => {
|
|
191
|
-
if (!cesdk.feature.unstable_isEnabled(FEATURE_ID, {
|
|
192
|
-
engine
|
|
193
|
-
})) {
|
|
194
|
-
return;
|
|
195
|
-
}
|
|
196
|
-
const [id] = engine.block.findAllSelected();
|
|
197
|
-
const metadata = getBGRemovalMetadata(cesdk, id);
|
|
198
|
-
const isActive = metadata.status === "PROCESSED_WITHOUT_BG";
|
|
199
|
-
const isLoading = metadata.status === "PROCESSING";
|
|
200
|
-
const isDisabled = metadata.status === "PENDING" || metadata.status === "PROCESSING";
|
|
201
|
-
let loadingProgress;
|
|
202
|
-
if (isLoading && metadata.progress) {
|
|
203
|
-
const { key, current, total } = metadata.progress;
|
|
204
|
-
if (key === "compute:inference") {
|
|
205
|
-
loadingProgress = void 0;
|
|
206
|
-
} else if (key.startsWith("fetch:/models/")) {
|
|
207
|
-
loadingProgress = current / total * 50;
|
|
208
|
-
} else if (key.startsWith("fetch:/onnxruntime-web/")) {
|
|
209
|
-
loadingProgress = 50 + current / total * 50;
|
|
210
|
-
} else {
|
|
211
|
-
loadingProgress = void 0;
|
|
212
|
-
}
|
|
213
|
-
}
|
|
214
|
-
Button(CANVAS_MENU_COMPONENT_BUTTON_ID, {
|
|
215
|
-
label: "BG Removal",
|
|
216
|
-
icon: "@imgly/icons/BGRemove",
|
|
217
|
-
isActive,
|
|
218
|
-
isLoading,
|
|
219
|
-
isDisabled,
|
|
220
|
-
loadingProgress,
|
|
221
|
-
onClick: () => {
|
|
222
|
-
switch (metadata.status) {
|
|
223
|
-
case "IDLE":
|
|
224
|
-
case "ERROR": {
|
|
225
|
-
setBGRemovalMetadata(cesdk, id, {
|
|
226
|
-
status: "PENDING"
|
|
227
|
-
});
|
|
228
|
-
break;
|
|
229
|
-
}
|
|
230
|
-
case "PROCESSED_WITHOUT_BG":
|
|
231
|
-
case "PROCESSED_WITH_BG": {
|
|
232
|
-
toggleBackgroundRemovalData(cesdk, id);
|
|
233
|
-
break;
|
|
234
|
-
}
|
|
235
|
-
default: {
|
|
236
|
-
}
|
|
237
|
-
}
|
|
238
|
-
}
|
|
239
|
-
});
|
|
240
|
-
}
|
|
241
|
-
);
|
|
242
|
-
}
|
|
243
|
-
|
|
244
|
-
// src/enableFeatures.ts
|
|
245
|
-
function enableFeatures(cesdk) {
|
|
246
|
-
cesdk.feature.unstable_enable(FEATURE_ID, ({ engine }) => {
|
|
247
|
-
const selectedIds = engine.block.findAllSelected();
|
|
248
|
-
if (selectedIds.length !== 1) {
|
|
249
|
-
return false;
|
|
250
|
-
}
|
|
251
|
-
const [selectedId] = selectedIds;
|
|
252
|
-
if (cesdk.engine.block.hasFill(selectedId)) {
|
|
253
|
-
const fillId = cesdk.engine.block.getFill(selectedId);
|
|
254
|
-
const fillType = cesdk.engine.block.getType(fillId);
|
|
255
|
-
if (fillType !== "//ly.img.ubq/fill/image") {
|
|
256
|
-
return false;
|
|
257
|
-
}
|
|
258
|
-
const fileUri = engine.block.getString(fillId, "fill/image/imageFileURI");
|
|
259
|
-
const sourceSet = engine.block.getSourceSet(
|
|
260
|
-
fillId,
|
|
261
|
-
"fill/image/sourceSet"
|
|
262
|
-
);
|
|
263
|
-
if (sourceSet.length > 0 || fileUri !== "")
|
|
264
|
-
return true;
|
|
265
|
-
const metadata = getBGRemovalMetadata(cesdk, selectedId);
|
|
266
|
-
return metadata.status === "PROCESSING";
|
|
267
|
-
}
|
|
268
|
-
return false;
|
|
269
|
-
});
|
|
270
|
-
}
|
|
271
|
-
|
|
272
|
-
// src/processBackgroundRemoval.ts
|
|
273
|
-
import {
|
|
274
|
-
segmentForeground,
|
|
275
|
-
applySegmentationMask
|
|
276
|
-
} from "@imgly/background-removal";
|
|
277
|
-
import throttle from "lodash/throttle";
|
|
278
|
-
async function processBackgroundRemoval(cesdk, blockId, configuration) {
|
|
279
|
-
const blockApi = cesdk.engine.block;
|
|
280
|
-
if (!blockApi.hasFill(blockId))
|
|
281
|
-
throw new Error("Block has no fill to remove the background from");
|
|
282
|
-
const fillId = blockApi.getFill(blockId);
|
|
283
|
-
const initialSourceSet = blockApi.getSourceSet(
|
|
284
|
-
fillId,
|
|
285
|
-
"fill/image/sourceSet"
|
|
286
|
-
);
|
|
287
|
-
const initialImageFileURI = blockApi.getString(
|
|
288
|
-
fillId,
|
|
289
|
-
"fill/image/imageFileURI"
|
|
290
|
-
);
|
|
291
|
-
try {
|
|
292
|
-
blockApi.setString(fillId, "fill/image/imageFileURI", "");
|
|
293
|
-
blockApi.setSourceSet(fillId, "fill/image/sourceSet", []);
|
|
294
|
-
const metadata = getBGRemovalMetadata(cesdk, blockId);
|
|
295
|
-
setBGRemovalMetadata(cesdk, blockId, {
|
|
296
|
-
...metadata,
|
|
297
|
-
version: "0.1.0",
|
|
298
|
-
initialSourceSet,
|
|
299
|
-
initialImageFileURI,
|
|
300
|
-
blockId,
|
|
301
|
-
fillId,
|
|
302
|
-
status: "PROCESSING"
|
|
303
|
-
});
|
|
304
|
-
const uriToProcess = (
|
|
305
|
-
// Source sets have priority in the engine
|
|
306
|
-
initialSourceSet.length > 0 ? (
|
|
307
|
-
// Choose the highest resolution image in the source set
|
|
308
|
-
initialSourceSet.sort(
|
|
309
|
-
(a, b) => b.width * b.height - a.height * a.width
|
|
310
|
-
)[0].uri
|
|
311
|
-
) : initialImageFileURI
|
|
312
|
-
);
|
|
313
|
-
const mask = await segmentForeground(uriToProcess, configuration);
|
|
314
|
-
if (initialSourceSet.length > 0) {
|
|
315
|
-
const uploaded = await maskSourceSet(
|
|
316
|
-
cesdk,
|
|
317
|
-
blockId,
|
|
318
|
-
initialSourceSet,
|
|
319
|
-
mask,
|
|
320
|
-
configuration
|
|
321
|
-
);
|
|
322
|
-
if (uploaded == null)
|
|
323
|
-
return;
|
|
324
|
-
if (uploaded.every((url) => url == null)) {
|
|
325
|
-
throw new Error("Could not upload any BG removed image");
|
|
326
|
-
}
|
|
327
|
-
const newSourceSet = initialSourceSet.map((source, index) => {
|
|
328
|
-
return {
|
|
329
|
-
...source,
|
|
330
|
-
uri: uploaded[index]
|
|
331
|
-
};
|
|
332
|
-
});
|
|
333
|
-
setBGRemovalMetadata(cesdk, blockId, {
|
|
334
|
-
version: "0.1.0",
|
|
335
|
-
initialSourceSet,
|
|
336
|
-
initialImageFileURI,
|
|
337
|
-
blockId,
|
|
338
|
-
fillId,
|
|
339
|
-
status: "PROCESSED_WITHOUT_BG",
|
|
340
|
-
removedBackground: newSourceSet
|
|
341
|
-
});
|
|
342
|
-
blockApi.setSourceSet(fillId, "fill/image/sourceSet", newSourceSet);
|
|
343
|
-
} else {
|
|
344
|
-
const uploaded = await maskSourceSet(
|
|
345
|
-
cesdk,
|
|
346
|
-
blockId,
|
|
347
|
-
[{ uri: uriToProcess }],
|
|
348
|
-
mask,
|
|
349
|
-
configuration
|
|
350
|
-
);
|
|
351
|
-
if (uploaded == null)
|
|
352
|
-
return;
|
|
353
|
-
const uploadedUrl = uploaded[0];
|
|
354
|
-
if (uploadedUrl == null) {
|
|
355
|
-
throw new Error("Could not upload BG removed image");
|
|
356
|
-
}
|
|
357
|
-
setBGRemovalMetadata(cesdk, blockId, {
|
|
358
|
-
version: "0.1.0",
|
|
359
|
-
initialSourceSet,
|
|
360
|
-
initialImageFileURI,
|
|
361
|
-
blockId,
|
|
362
|
-
fillId,
|
|
363
|
-
status: "PROCESSED_WITHOUT_BG",
|
|
364
|
-
removedBackground: uploadedUrl
|
|
365
|
-
});
|
|
366
|
-
blockApi.setString(fillId, "fill/image/imageFileURI", uploadedUrl);
|
|
367
|
-
}
|
|
368
|
-
cesdk.engine.editor.addUndoStep();
|
|
369
|
-
} catch (error) {
|
|
370
|
-
if (cesdk.engine.block.isValid(blockId)) {
|
|
371
|
-
setBGRemovalMetadata(cesdk, blockId, {
|
|
372
|
-
version: "0.1.0",
|
|
373
|
-
initialSourceSet,
|
|
374
|
-
initialImageFileURI,
|
|
375
|
-
blockId,
|
|
376
|
-
fillId,
|
|
377
|
-
status: "ERROR"
|
|
378
|
-
});
|
|
379
|
-
recoverInitialImageData(cesdk, blockId);
|
|
380
|
-
}
|
|
381
|
-
console.log(error);
|
|
382
|
-
}
|
|
383
|
-
}
|
|
384
|
-
async function maskSourceSet(cesdk, blockId, urisOrSources, mask, configurationFromArgs) {
|
|
385
|
-
const configuration = {
|
|
386
|
-
...configurationFromArgs,
|
|
387
|
-
progress: throttle((key, current, total) => {
|
|
388
|
-
const metadataDuringProgress = getBGRemovalMetadata(cesdk, blockId);
|
|
389
|
-
if (metadataDuringProgress.status !== "PROCESSING" || !isMetadataConsistent(cesdk, blockId))
|
|
390
|
-
return;
|
|
391
|
-
configurationFromArgs.progress?.(key, current, total);
|
|
392
|
-
setBGRemovalMetadata(cesdk, blockId, {
|
|
393
|
-
...metadataDuringProgress,
|
|
394
|
-
progress: { key, current, total }
|
|
395
|
-
});
|
|
396
|
-
}, 100)
|
|
397
|
-
};
|
|
398
|
-
const masked = await Promise.all(
|
|
399
|
-
urisOrSources.map(async (source) => {
|
|
400
|
-
const blob = await applySegmentationMask(source.uri, mask, configuration);
|
|
401
|
-
return [blob, source];
|
|
402
|
-
})
|
|
403
|
-
);
|
|
404
|
-
if (getBGRemovalMetadata(cesdk, blockId).status !== "PROCESSING" || !isMetadataConsistent(cesdk, blockId))
|
|
405
|
-
return;
|
|
406
|
-
const uploaded = await Promise.all(
|
|
407
|
-
masked.map(async ([blob, source]) => {
|
|
408
|
-
const pathname = new URL(source.uri).pathname;
|
|
409
|
-
const parts = pathname.split("/");
|
|
410
|
-
const filename = parts[parts.length - 1];
|
|
411
|
-
const uploadedAssets = await cesdk.unstable_upload(
|
|
412
|
-
new File([blob], filename, { type: blob.type }),
|
|
413
|
-
() => {
|
|
414
|
-
}
|
|
415
|
-
);
|
|
416
|
-
const url = uploadedAssets.meta?.uri;
|
|
417
|
-
if (url == null) {
|
|
418
|
-
throw new Error("Could not upload BG removed image");
|
|
419
|
-
}
|
|
420
|
-
return [url, source];
|
|
421
|
-
})
|
|
422
|
-
);
|
|
423
|
-
if (getBGRemovalMetadata(cesdk, blockId).status !== "PROCESSING" || !isMetadataConsistent(cesdk, blockId))
|
|
424
|
-
return;
|
|
425
|
-
return uploaded.map(([url]) => url);
|
|
426
|
-
}
|
|
427
|
-
|
|
428
|
-
// src/plugin.ts
|
|
429
|
-
var plugin_default = (pluginConfiguration = {}) => {
|
|
430
|
-
const backgroundRemovalConfiguration = pluginConfiguration?.backgroundRemoval ?? {};
|
|
431
|
-
return {
|
|
432
|
-
initialize() {
|
|
433
|
-
},
|
|
434
|
-
update() {
|
|
435
|
-
},
|
|
436
|
-
initializeUserInterface({ cesdk }) {
|
|
437
|
-
cesdk.engine.event.subscribe([], async (events) => {
|
|
438
|
-
events.forEach((e) => {
|
|
439
|
-
const id = e.block;
|
|
440
|
-
if (!cesdk.engine.block.isValid(id) || !cesdk.engine.block.hasMetadata(id, BG_REMOVAL_ID)) {
|
|
441
|
-
return;
|
|
442
|
-
}
|
|
443
|
-
if (e.type === "Created") {
|
|
444
|
-
const metadata = getBGRemovalMetadata(cesdk, id);
|
|
445
|
-
if (isDuplicate(cesdk, id, metadata)) {
|
|
446
|
-
fixDuplicateMetadata(cesdk, id);
|
|
447
|
-
}
|
|
448
|
-
} else if (e.type === "Updated") {
|
|
449
|
-
handleUpdateEvent(cesdk, id, backgroundRemovalConfiguration);
|
|
450
|
-
}
|
|
451
|
-
});
|
|
452
|
-
});
|
|
453
|
-
registerComponents(cesdk);
|
|
454
|
-
enableFeatures(cesdk);
|
|
455
|
-
}
|
|
456
|
-
};
|
|
457
|
-
};
|
|
458
|
-
async function handleUpdateEvent(cesdk, blockId, configuration) {
|
|
459
|
-
const metadata = getBGRemovalMetadata(cesdk, blockId);
|
|
460
|
-
switch (metadata.status) {
|
|
461
|
-
case "PENDING": {
|
|
462
|
-
if (cesdk.feature.unstable_isEnabled(FEATURE_ID, {
|
|
463
|
-
engine: cesdk.engine
|
|
464
|
-
})) {
|
|
465
|
-
processBackgroundRemoval(cesdk, blockId, configuration);
|
|
466
|
-
}
|
|
467
|
-
break;
|
|
468
|
-
}
|
|
469
|
-
case "PROCESSING":
|
|
470
|
-
case "PROCESSED_WITH_BG":
|
|
471
|
-
case "PROCESSED_WITHOUT_BG": {
|
|
472
|
-
if (!isMetadataConsistent(cesdk, blockId)) {
|
|
473
|
-
clearBGRemovalMetadata(cesdk, blockId);
|
|
474
|
-
}
|
|
475
|
-
break;
|
|
476
|
-
}
|
|
477
|
-
default: {
|
|
478
|
-
}
|
|
479
|
-
}
|
|
480
|
-
}
|
|
481
|
-
|
|
482
|
-
// src/index.ts
|
|
483
|
-
var Plugin = (pluginConfiguration) => ({
|
|
484
|
-
name: BG_REMOVAL_ID,
|
|
485
|
-
version: "0.1.0",
|
|
486
|
-
...plugin_default(pluginConfiguration)
|
|
487
|
-
});
|
|
488
|
-
var src_default = Plugin;
|
|
489
|
-
export {
|
|
490
|
-
src_default as default
|
|
491
|
-
};
|
|
1
|
+
import p from"lodash/isEqual";var w=class{constructor(e,i){this.cesdk=e,this.key=i}hasData(e){return this.cesdk.engine.block.isValid(e)&&this.cesdk.engine.block.hasMetadata(e,this.key)}get(e){if(this.hasData(e))return JSON.parse(this.cesdk.engine.block.getMetadata(e,this.key))}set(e,i){this.cesdk.engine.block.setMetadata(e,this.key,JSON.stringify(i))}clear(e){this.cesdk.engine.block.hasMetadata(e,this.key)&&this.cesdk.engine.block.removeMetadata(e,this.key)}},D=w,U=class extends D{get(e){return super.get(e)??{status:"IDLE"}}isDuplicate(e){if(!this.cesdk.engine.block.isValid(e))return!1;let i=this.get(e);if(i.status==="IDLE"||i.status==="PENDING"||i.status==="ERROR"||!this.cesdk.engine.block.hasFill(e))return!1;let t=this.cesdk.engine.block.getFill(e);return!(i.blockId===e||i.fillId===t)}fixDuplicate(e){let i=this.cesdk.engine.block.getFill(e),t=this.get(e);t.status==="IDLE"||t.status==="PENDING"||t.status==="ERROR"||(this.set(e,{...t,blockId:e,fillId:i}),t.status==="PROCESSING"&&(this.recoverInitialImageData(e),this.clear(e)))}isConsistent(e){if(!this.cesdk.engine.block.isValid(e))return!1;let i=this.get(e);if(i.status==="IDLE"||i.status==="PENDING")return!0;if(!this.cesdk.engine.block.hasFill(e))return!1;let t=this.cesdk.engine.block.getFill(e);if(t==null||e!==i.blockId||t!==i.fillId)return!1;let a=this.cesdk.engine.block.getSourceSet(t,"fill/image/sourceSet"),l=this.cesdk.engine.block.getString(t,"fill/image/imageFileURI");if(a.length===0&&!l&&i.status==="PROCESSING")return!0;if(a?.length>0){let s=i.initialSourceSet;if(i.status==="PROCESSED"){if(!p(a,i.processed)&&!p(a,s))return!1}else if(!p(a,s))return!1}else if(i.status==="PROCESSED"){if(l!==i.initialImageFileURI&&l!==i.processed)return!1}else if(l!==i.initialImageFileURI)return!1;return!0}recoverInitialImageData(e){if(!this.cesdk.engine.block.hasFill(e))return;let i=this.get(e);if(i.status==="PENDING"||i.status==="IDLE")return;let t=i.initialSourceSet,a=i.initialImageFileURI,l=i.initialPreviewFileURI,s=this.getValidFill(e,i);s!=null&&(a&&this.cesdk.engine.block.setString(s,"fill/image/imageFileURI",a),l&&this.cesdk.engine.block.setString(s,"fill/image/previewFileURI",l),t.length>0&&this.cesdk.engine.block.setSourceSet(s,"fill/image/sourceSet",t))}getValidFill(e,i){if(!this.cesdk.engine.block.isValid(e)||!this.cesdk.engine.block.hasFill(e)||e!==i.blockId)return;let t=this.cesdk.engine.block.getFill(e);if(t===i.fillId)return t}},I=U;async function S(e,i,t,a,l){let s=e.engine.block;if(!s.hasFill(i))throw new Error("Block has no fill to process");let n=s.getFill(i),r=s.getSourceSet(n,"fill/image/sourceSet"),u=s.getString(n,"fill/image/imageFileURI"),d=s.getString(n,"fill/image/previewFileURI");try{s.setString(n,"fill/image/imageFileURI",""),s.setSourceSet(n,"fill/image/sourceSet",[]),t.set(i,{...t.get(i),version:"0.0.0",initialSourceSet:r,initialImageFileURI:u,initialPreviewFileURI:d,blockId:i,fillId:n,status:"PROCESSING"});let o=r.sort((c,f)=>f.width*f.height-c.height*c.width),g=r.length>0?o[0].uri:u;if(d||s.setString(n,"fill/image/previewFileURI",g),r.length>0){let c=await a(r);if(t.get(i).status!=="PROCESSING"||!t.isConsistent(i)||t.get(i).status!=="PROCESSING"||!t.isConsistent(i)||c==null)return;if(c.every(f=>f==null))throw new Error("Empty source set after processing fill");t.set(i,{version:"0.0.0",initialSourceSet:r,initialImageFileURI:u,initialPreviewFileURI:d,blockId:i,fillId:n,status:"PROCESSED",processed:c}),s.setSourceSet(n,"fill/image/sourceSet",c),s.setString(n,"fill/image/previewFileURI","")}else{let c=await l(g);if(t.get(i).status!=="PROCESSING"||!t.isConsistent(i)||t.get(i).status!=="PROCESSING"||!t.isConsistent(i))return;if(c==null)throw new Error("Could not upload fill processed data");t.set(i,{version:"0.0.0",initialSourceSet:r,initialImageFileURI:u,initialPreviewFileURI:d,blockId:i,fillId:n,status:"PROCESSED",processed:c}),s.setString(n,"fill/image/imageFileURI",c),s.setString(n,"fill/image/previewFileURI","")}e.engine.editor.addUndoStep()}catch(o){e.engine.block.isValid(i)&&(t.set(i,{version:"0.0.0",initialSourceSet:r,initialImageFileURI:u,initialPreviewFileURI:d,blockId:i,fillId:n,status:"ERROR"}),t.recoverInitialImageData(i)),o!=null&&typeof o=="object"&&"message"in o&&typeof o.message=="string"&&e.ui.showNotification({type:"error",message:o.message}),console.log(o)}}function h(e){return`${e}.fillProcessing.feature`}function N(e){return`${e}.fillProcessing.canvasMenu`}function O(e){return`plugin.${e}.fillProcessing.canvasMenu.button.label`}function b(e,{pluginId:i,process:t}){let a=h(i),l=new I(e,i);return G(e,l,a),e.engine.event.subscribe([],async s=>{s.forEach(n=>{let r=n.block;if(!(!e.engine.block.isValid(r)||!l.hasData(r))){if(n.type==="Created")l.isDuplicate(r)&&l.fixDuplicate(r);else if(n.type==="Updated")switch(l.get(r).status){case"PENDING":{e.feature.unstable_isEnabled(a,{engine:e.engine})&&e.engine.block.isAllowedByScope(r,"fill/change")&&t(r,l);break}case"PROCESSING":case"PROCESSED":{l.isConsistent(r)||l.clear(r);break}default:}}})}),{featureId:a}}function G(e,i,t){e.feature.unstable_enable(t,({engine:a})=>{let l=a.block.findAllSelected();if(l.length!==1)return!1;let[s]=l;if(!e.engine.block.isVisible(s))return!1;if(e.engine.block.hasFill(s)){if(e.engine.block.getKind(s)==="sticker")return!1;let n=e.engine.block.getFill(s);if(e.engine.block.getType(n)!=="//ly.img.ubq/fill/image")return!1;let r=a.block.getString(n,"fill/image/imageFileURI");return a.block.getSourceSet(n,"fill/image/sourceSet").length>0||r!==""?!0:i.get(s).status==="PROCESSING"}return!1})}function R(e,i){let{pluginId:t,locations:a}=i,l=new I(e,t),s=N(t),n=O(t),r=h(t);return a?.includes("canvasMenu")&&e.ui.unstable_setCanvasMenuOrder([s,...e.ui.unstable_getCanvasMenuOrder()]),e.ui.unstable_registerComponent(s,({builder:{Button:u},engine:d})=>{if(!e.feature.unstable_isEnabled(r,{engine:d}))return;let[o]=d.block.findAllSelected();if(!e.engine.block.isAllowedByScope(o,"fill/change"))return;let g=l.get(o),c=g.status==="PROCESSING",f=g.status==="PENDING"||g.status==="PROCESSING",k;if(c&&g.progress){let{current:F,total:y}=g.progress;k=F/y*100}let C=`${s}.button`;u(C,{label:n,icon:i.icon,isLoading:c,isDisabled:f,loadingProgress:k,onClick:()=>{(g.status==="IDLE"||g.status==="ERROR"||g.status==="PROCESSED")&&l.set(o,{status:"PENDING"})}})}),{canvasMenuComponentId:s,translationsKeys:{canvasMenuLabel:n}}}import{applySegmentationMask as M,removeBackground as B,segmentForeground as L}from"@imgly/background-removal";import x from"lodash/throttle";async function P(e,i,t,a){switch(a.type){case"@imgly/background-removal":{let l=a.configuration??{},s={device:"gpu",...l,progress:x((n,r,u)=>{let d=t.get(i);d.status!=="PROCESSING"||!t.isConsistent(i)||(l.progress?.(n,r,u),t.set(i,{...d,progress:{key:n,current:r,total:u}}))},100)};S(e,i,t,async n=>{let r=n[0].uri,u=await L(r,l);return await Promise.all(n.map(async o=>{let g=await M(o.uri,u,s),c=await v(g,o.uri,e);return{...o,uri:c}}))},async n=>{let r=await B(n,s);return await v(r,n,e)});break}case"custom":{S(e,i,t,a.processSourceSet,a.processImageFileURI);break}default:throw new Error("Unknown background removal provider")}}async function v(e,i,t){let l=new URL(i).pathname.split("/"),s=l[l.length-1],r=(await t.unstable_upload(new File([e],s,{type:e.type}),()=>{})).meta?.uri;if(r==null)throw new Error("Could not upload processed fill");return r}var m="@imgly/plugin-background-removal-web",E=(e={})=>({initialize(){},update(){},initializeUserInterface({cesdk:i}){b(i,{pluginId:m,process:(a,l)=>{P(i,a,l,e.provider??{type:"@imgly/background-removal"})}});let{translationsKeys:t}=R(i,{pluginId:m,icon:"@imgly/icons/BGRemove",locations:e.ui?.locations});i.setTranslations({en:{[t.canvasMenuLabel]:"BG Removal"}})}});var V=e=>({name:m,version:"0.2.0-rc.0",...E(e)}),W=V;export{W as default};
|
|
492
2
|
//# sourceMappingURL=index.mjs.map
|